diff --git a/Cargo.lock b/Cargo.lock index 118f82b..6d72047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4210,6 +4210,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "metrics" version = "0.24.3" @@ -7822,6 +7831,7 @@ version = "0.4.7" dependencies = [ "async-trait", "bytes", + "chrono", "cid", "fjall", "flume 0.11.1", @@ -7837,12 +7847,16 @@ dependencies = [ "rand 0.8.5", "serde", "serde_ipld_dagcbor", + "serde_json", "sha2", "sqlx", "tempfile", + "thiserror 2.0.18", "tokio", "tracing", + "tranquil-db-traits", "tranquil-repo", + "tranquil-types", "xxhash-rust", ] diff --git a/crates/tranquil-store/Cargo.toml b/crates/tranquil-store/Cargo.toml index 984cde7..f8d18a1 100644 --- a/crates/tranquil-store/Cargo.toml +++ b/crates/tranquil-store/Cargo.toml @@ -17,6 +17,11 @@ tokio = { workspace = true, features = ["sync", "rt"] } bytes = "1" memmap2 = "0.9" tracing = { workspace = true } +chrono = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tranquil-db-traits = { workspace = true } +tranquil-types = { workspace = true } jacquard-repo = { workspace = true } cid = { workspace = true } multihash = { workspace = true } @@ -42,3 +47,19 @@ tikv-jemallocator = "0.6" [[bench]] name = "blockstore" harness = false + +[[bench]] +name = "eventlog" +harness = false + +[[bench]] +name = "metastore" +harness = false + +[[bench]] +name = "metastore_scale" +harness = false + +[[bench]] +name = "profile_reads" +harness = false diff --git a/crates/tranquil-store/benches/eventlog.rs b/crates/tranquil-store/benches/eventlog.rs new file mode 100644 index 0000000..17d2978 --- /dev/null +++ b/crates/tranquil-store/benches/eventlog.rs @@ -0,0 +1,959 @@ +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use chrono::Utc; +use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent}; +use tranquil_types::Did; + +use tranquil_store::RealIO; +use tranquil_store::eventlog::{EventLog, EventLogConfig, EventSequence}; + +fn make_did(index: usize) -> Did { + let suffix: String = format!("{index:024x}"); + Did::new(format!("did:plc:{suffix}")).unwrap() +} + +fn make_event(index: usize) -> SequencedEvent { + let ops_size = match index % 4 { + 0 => 64, + 1 => 256, + 2 => 1024, + _ => 4096, + }; + + let ops_payload: String = (0..ops_size) + .map(|i| ((index.wrapping_mul(31).wrapping_add(i)) % 26 + 97) as u8 as char) + .collect(); + + SequencedEvent { + seq: SequenceNumber::from_raw(i64::try_from(index + 1).expect("event index overflow")), + did: make_did(index % 10_000), + created_at: Utc::now(), + event_type: match index % 4 { + 0 => RepoEventType::Commit, + 1 => RepoEventType::Identity, + 2 => RepoEventType::Account, + _ => RepoEventType::Sync, + }, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: Some(serde_json::json!({ "data": ops_payload })), + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: None, + } +} + +fn estimated_payload_size(event: &SequencedEvent) -> usize { + tranquil_store::eventlog::encode_payload(event).len() +} + +struct LatencyStats { + p50: Duration, + p95: Duration, + p99: Duration, + max: Duration, + mean: Duration, +} + +fn compute_stats(durations: &mut [Duration]) -> Option { + if durations.is_empty() { + return None; + } + durations.sort(); + let len = durations.len(); + let sum: Duration = durations.iter().sum(); + let divisor = u32::try_from(len).unwrap_or(u32::MAX); + let last = len - 1; + Some(LatencyStats { + p50: durations[last * 50 / 100], + p95: durations[last * 95 / 100], + p99: durations[last * 99 / 100], + max: durations[last], + mean: sum / divisor, + }) +} + +fn format_latency(stats: Option<&LatencyStats>) -> String { + match stats { + Some(s) => format!( + " | p50={:?} p95={:?} p99={:?} max={:?} mean={:?}", + s.p50, s.p95, s.p99, s.max, s.mean + ), + None => String::new(), + } +} + +fn open_eventlog(dir: &Path) -> EventLog { + let segments_dir = dir.join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap() +} + +fn bench_sequential_append(event_count: usize) { + println!("-- sequential append: {event_count} events --"); + let dir = tempfile::TempDir::new().unwrap(); + let log = open_eventlog(dir.path()); + + let events: Vec = (0..event_count).map(make_event).collect(); + let total_bytes: usize = events.iter().map(estimated_payload_size).sum(); + let mut latencies = Vec::with_capacity(event_count); + + let start = Instant::now(); + events.iter().enumerate().for_each(|(i, event)| { + let t = Instant::now(); + log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event) + .unwrap(); + if (i + 1) % 256 == 0 { + log.sync().unwrap(); + } + latencies.push(t.elapsed()); + }); + log.sync().unwrap(); + let elapsed = start.elapsed(); + + let lat = format_latency(compute_stats(&mut latencies).as_ref()); + println!( + "{:.0} events/sec, {:.1} MB/sec, {:.1}ms{lat}", + event_count as f64 / elapsed.as_secs_f64(), + total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0), + elapsed.as_secs_f64() * 1000.0, + ); + let _ = log.shutdown(); +} + +fn bench_concurrent_producers(event_count: usize, producers: usize) { + println!("-- {producers} concurrent producers, {event_count} total events --"); + let dir = tempfile::TempDir::new().unwrap(); + let log = Arc::new(open_eventlog(dir.path())); + + let events_per_producer = event_count / producers; + let actual_count = events_per_producer * producers; + let avg_payload: usize = (0..4) + .map(|i| estimated_payload_size(&make_event(i))) + .sum::() + / 4; + + let start = Instant::now(); + + let handles: Vec<_> = (0..producers) + .map(|pid| { + let log = Arc::clone(&log); + std::thread::spawn(move || { + let mut latencies = Vec::with_capacity(events_per_producer); + (0..events_per_producer).for_each(|i| { + let global = pid * events_per_producer + i; + let event = make_event(global); + let t = Instant::now(); + log.append_and_sync(&make_did(global % 10_000), RepoEventType::Commit, &event) + .unwrap(); + latencies.push(t.elapsed()); + }); + latencies + }) + }) + .collect(); + + let mut all_latencies: Vec = handles + .into_iter() + .flat_map(|h| h.join().unwrap()) + .collect(); + let elapsed = start.elapsed(); + + let total_bytes = actual_count * avg_payload; + let lat = format_latency(compute_stats(&mut all_latencies).as_ref()); + println!( + "{:.0} events/sec, {:.1} MB/sec, {:.1}ms{lat}", + actual_count as f64 / elapsed.as_secs_f64(), + total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0), + elapsed.as_secs_f64() * 1000.0, + ); + let _ = log.shutdown(); +} + +fn bench_batch_append(event_count: usize, batch_size: usize) { + println!("-- batch append: {event_count} events, batch_size={batch_size} --"); + let dir = tempfile::TempDir::new().unwrap(); + let log = open_eventlog(dir.path()); + + let events: Vec = (0..event_count).map(make_event).collect(); + let dids: Vec = (0..event_count).map(|i| make_did(i % 10_000)).collect(); + let total_bytes: usize = events.iter().map(estimated_payload_size).sum(); + let mut batch_latencies = Vec::with_capacity(event_count / batch_size + 1); + + let start = Instant::now(); + events + .chunks(batch_size) + .enumerate() + .for_each(|(chunk_idx, chunk)| { + let base = chunk_idx * batch_size; + let batch: Vec<(&Did, RepoEventType, &SequencedEvent)> = chunk + .iter() + .enumerate() + .map(|(j, event)| (&dids[base + j], RepoEventType::Commit, event)) + .collect(); + let t = Instant::now(); + log.append_batch(batch).unwrap(); + log.sync().unwrap(); + batch_latencies.push(t.elapsed()); + }); + let elapsed = start.elapsed(); + + let lat = format_latency(compute_stats(&mut batch_latencies).as_ref()); + println!( + "{:.0} events/sec, {:.1} MB/sec, {:.1}ms{lat}", + event_count as f64 / elapsed.as_secs_f64(), + total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0), + elapsed.as_secs_f64() * 1000.0, + ); + let _ = log.shutdown(); +} + +fn bench_rotation_under_load(event_count: usize) { + println!("-- rotation: {event_count} events, 256KB segments --"); + let dir = tempfile::TempDir::new().unwrap(); + let segments_dir = dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + let log = EventLog::open( + EventLogConfig { + segments_dir, + max_segment_size: 256 * 1024, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let events: Vec = (0..event_count).map(make_event).collect(); + let append_latencies = Vec::with_capacity(event_count); + let rotation_latencies = Vec::new(); + + let start = Instant::now(); + let (mut append_latencies, mut rotation_latencies, _) = events.iter().enumerate().fold( + (append_latencies, rotation_latencies, false), + |(mut appends, mut rotations, fd_limited), (i, event)| { + let t = Instant::now(); + log.append_and_sync(&make_did(i % 10_000), RepoEventType::Commit, event) + .unwrap(); + appends.push(t.elapsed()); + + match fd_limited { + true => (appends, rotations, true), + false => { + let rt = Instant::now(); + match log.maybe_rotate() { + Ok(true) => { + rotations.push(rt.elapsed()); + (appends, rotations, false) + } + Ok(false) => (appends, rotations, false), + Err(e) => { + println!("fd limit hit at {} segments: {e}", log.segment_count()); + (appends, rotations, true) + } + } + } + } + }, + ); + let elapsed = start.elapsed(); + + let append_lat = format_latency(compute_stats(&mut append_latencies).as_ref()); + let rotation_lat = format_latency(compute_stats(&mut rotation_latencies).as_ref()); + println!( + "{:.0} events/sec, {} segments, {} rotations, {:.1}ms", + event_count as f64 / elapsed.as_secs_f64(), + log.segment_count(), + rotation_latencies.len(), + elapsed.as_secs_f64() * 1000.0, + ); + println!("append{append_lat}"); + println!("rotation{rotation_lat}"); + let _ = log.shutdown(); +} + +fn scan_all_events(log: &EventLog, batch_size: usize) -> usize { + scan_all_events_from(log, EventSequence::BEFORE_ALL, batch_size, 0) +} + +fn scan_all_events_from( + log: &EventLog, + cursor: EventSequence, + batch_size: usize, + accumulated: usize, +) -> usize { + let batch = log.get_events_since(cursor, batch_size).unwrap(); + match batch.last() { + None => accumulated, + Some(last) => { + let next_cursor = EventSequence::new(u64::try_from(last.seq.as_i64()).unwrap()); + scan_all_events_from(log, next_cursor, batch_size, accumulated + batch.len()) + } + } +} + +fn bench_sequential_scan(event_count: usize) { + println!("-- sequential scan: {event_count} events --"); + let dir = tempfile::TempDir::new().unwrap(); + let log = open_eventlog(dir.path()); + + let events: Vec = (0..event_count).map(make_event).collect(); + let total_bytes: usize = events.iter().map(estimated_payload_size).sum(); + + events.iter().enumerate().for_each(|(i, event)| { + log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event) + .unwrap(); + }); + log.sync().unwrap(); + + let start = Instant::now(); + let read_count = scan_all_events(&log, 4096); + let elapsed = start.elapsed(); + + println!( + "{:.0} events/sec, {:.1} MB/sec, {read_count} events, {:.1}ms", + read_count as f64 / elapsed.as_secs_f64(), + total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0), + elapsed.as_secs_f64() * 1000.0, + ); + let _ = log.shutdown(); +} + +fn bench_parallel_readers(event_count: usize, readers: usize) { + println!("-- parallel readers: {event_count} events, {readers} readers --"); + let dir = tempfile::TempDir::new().unwrap(); + let log = Arc::new(open_eventlog(dir.path())); + + let events: Vec = (0..event_count).map(make_event).collect(); + + events.iter().enumerate().for_each(|(i, event)| { + log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event) + .unwrap(); + }); + log.sync().unwrap(); + + let total_read = Arc::new(AtomicU64::new(0)); + let batch_size = 4096; + + let start = Instant::now(); + + let handles: Vec<_> = (0..readers) + .map(|_| { + let log = Arc::clone(&log); + let total_read = Arc::clone(&total_read); + std::thread::spawn(move || { + let count = scan_all_events(&log, batch_size) as u64; + total_read.fetch_add(count, Ordering::Relaxed); + }) + }) + .collect(); + + handles.into_iter().for_each(|h| h.join().unwrap()); + let elapsed = start.elapsed(); + + let total = total_read.load(Ordering::Relaxed); + let avg_payload: usize = (0..4) + .map(|i| estimated_payload_size(&make_event(i))) + .sum::() + / 4; + println!( + "{:.0} total events/sec across {readers} readers ({:.0} per reader)", + total as f64 / elapsed.as_secs_f64(), + (total as f64 / readers as f64) / elapsed.as_secs_f64(), + ); + println!( + "aggregate {:.1} MB/sec, {:.1}ms", + (total as f64 * avg_payload as f64) / elapsed.as_secs_f64() / (1024.0 * 1024.0), + elapsed.as_secs_f64() * 1000.0, + ); + let _ = log.shutdown(); +} + +fn bench_stampede(event_count: usize, producers: usize, readers: usize, subscribers: usize) { + println!( + "-- stampede: {event_count} events, {producers} producers, {readers} readers, {subscribers} subscribers --" + ); + let dir = tempfile::TempDir::new().unwrap(); + let log = Arc::new(open_eventlog(dir.path())); + + let events_per_producer = event_count / producers; + let actual_events = events_per_producer * producers; + + let writes_done = Arc::new(AtomicBool::new(false)); + let total_written = Arc::new(AtomicU64::new(0)); + let total_read = Arc::new(AtomicU64::new(0)); + let total_subscribed = Arc::new(AtomicU64::new(0)); + + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + let start = Instant::now(); + + let subscriber_handles: Vec<_> = (0..subscribers) + .map(|_| { + let log = Arc::clone(&log); + let writes_done = Arc::clone(&writes_done); + let total_subscribed = Arc::clone(&total_subscribed); + let total_written = Arc::clone(&total_written); + rt.spawn(async move { + let mut sub = log.subscriber(EventSequence::BEFORE_ALL); + let mut count = 0u64; + loop { + match tokio::time::timeout(Duration::from_millis(100), sub.next()).await { + Ok(Some(_)) => { + count += 1; + } + Ok(None) => break, + Err(_) => { + if writes_done.load(Ordering::Acquire) { + let written = total_written.load(Ordering::Acquire); + if count >= written { + break; + } + match tokio::time::timeout(Duration::from_secs(2), sub.next()).await + { + Ok(Some(_)) => count += 1, + _ => break, + } + } + } + } + } + total_subscribed.fetch_add(count, Ordering::Relaxed); + }) + }) + .collect(); + + let writer_handles: Vec<_> = (0..producers) + .map(|pid| { + let log = Arc::clone(&log); + let total_written = Arc::clone(&total_written); + std::thread::spawn(move || { + let mut latencies = Vec::with_capacity(events_per_producer); + (0..events_per_producer).for_each(|i| { + let global = pid * events_per_producer + i; + let event = make_event(global); + let t = Instant::now(); + log.append_and_sync(&make_did(global % 10_000), RepoEventType::Commit, &event) + .unwrap(); + latencies.push(t.elapsed()); + total_written.fetch_add(1, Ordering::Release); + }); + latencies + }) + }) + .collect(); + + let reader_handles: Vec<_> = (0..readers) + .map(|_| { + let log = Arc::clone(&log); + let writes_done = Arc::clone(&writes_done); + let total_read = Arc::clone(&total_read); + std::thread::spawn(move || { + let mut cursor = EventSequence::BEFORE_ALL; + let mut count = 0u64; + loop { + let batch = log.get_events_since(cursor, 1024).unwrap(); + match batch.last() { + Some(last) => { + count += batch.len() as u64; + cursor = EventSequence::new(u64::try_from(last.seq.as_i64()).unwrap()); + } + None if writes_done.load(Ordering::Acquire) => { + let final_batch = log.get_events_since(cursor, 1024).unwrap(); + match final_batch.last() { + Some(last) => { + count += final_batch.len() as u64; + cursor = EventSequence::new( + u64::try_from(last.seq.as_i64()).unwrap(), + ); + } + None => break, + } + } + None => { + std::thread::yield_now(); + } + } + } + total_read.fetch_add(count, Ordering::Relaxed); + }) + }) + .collect(); + + let mut write_latencies: Vec = writer_handles + .into_iter() + .flat_map(|h| h.join().unwrap()) + .collect(); + let write_elapsed = start.elapsed(); + + writes_done.store(true, Ordering::Release); + + reader_handles.into_iter().for_each(|h| h.join().unwrap()); + let read_elapsed = start.elapsed(); + + rt.block_on(async { + let _ = tokio::time::timeout( + Duration::from_secs(10), + futures::future::join_all(subscriber_handles), + ) + .await; + }); + let total_elapsed = start.elapsed(); + + let reads = total_read.load(Ordering::Relaxed); + let subscribed = total_subscribed.load(Ordering::Relaxed); + + let write_lat = format_latency(compute_stats(&mut write_latencies).as_ref()); + println!( + "writes: {:.0} events/sec, {actual_events} events, {:.1}ms{write_lat}", + actual_events as f64 / write_elapsed.as_secs_f64(), + write_elapsed.as_secs_f64() * 1000.0, + ); + println!( + "reads: {:.0} events/sec, {readers} readers, {reads} events, {:.1}ms", + reads as f64 / read_elapsed.as_secs_f64(), + read_elapsed.as_secs_f64() * 1000.0, + ); + println!( + "subscribers: {subscribed} events across {subscribers} subscribers, {:.1}ms", + total_elapsed.as_secs_f64() * 1000.0, + ); + println!("segments: {}", log.segment_count()); + let _ = log.shutdown(); +} + +fn bench_broadcast_fanout(subscriber_count: usize) { + println!("-- broadcast fanout: 10000 events, {subscriber_count} subscribers --"); + let dir = tempfile::TempDir::new().unwrap(); + let log = Arc::new(open_eventlog(dir.path())); + + let event_count = 10_000usize; + + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8), + ) + .enable_all() + .build() + .unwrap(); + + rt.block_on(async { + let received_counts: Arc> = + Arc::new((0..subscriber_count).map(|_| AtomicU64::new(0)).collect()); + + let sub_handles: Vec<_> = (0..subscriber_count) + .map(|sub_id| { + let mut subscriber = log.subscriber(EventSequence::BEFORE_ALL); + let received_counts = Arc::clone(&received_counts); + tokio::spawn(async move { + let mut count = 0u64; + while count < event_count as u64 { + match tokio::time::timeout(Duration::from_secs(10), subscriber.next()).await + { + Ok(Some(_)) => count += 1, + _ => break, + } + } + received_counts[sub_id].store(count, Ordering::Relaxed); + }) + }) + .collect(); + + let log_writer = Arc::clone(&log); + let write_handle = tokio::task::spawn_blocking(move || { + let mut latencies = Vec::with_capacity(event_count); + (0..event_count).for_each(|i| { + let event = make_event(i); + let t = Instant::now(); + log_writer + .append_and_sync(&make_did(i % 10_000), RepoEventType::Commit, &event) + .unwrap(); + latencies.push(t.elapsed()); + }); + latencies + }); + + let mut write_latencies = write_handle.await.unwrap(); + + let _ = tokio::time::timeout( + Duration::from_secs(30), + futures::future::join_all(sub_handles), + ) + .await; + + let total_received: u64 = received_counts + .iter() + .map(|c| c.load(Ordering::Relaxed)) + .sum(); + let min_received = received_counts + .iter() + .map(|c| c.load(Ordering::Relaxed)) + .min() + .unwrap_or(0); + + let write_lat = format_latency(compute_stats(&mut write_latencies).as_ref()); + println!("write{write_lat}"); + println!( + "total received: {total_received}/{}, min per sub: {min_received}/{event_count}", + event_count as u64 * subscriber_count as u64, + ); + }); + + let _ = log.shutdown(); +} + +async fn bench_pg_write_throughput(event_count: usize, concurrency: usize) { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + println!("skipped, set DATABASE_URL to enable"); + return; + } + }; + + let max_conns = u32::try_from(concurrency) + .unwrap_or(u32::MAX) + .saturating_add(10); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(max_conns) + .acquire_timeout(Duration::from_secs(30)) + .connect(&database_url) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS bench_repo_seq ( + seq BIGSERIAL PRIMARY KEY, + did TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + event_type TEXT NOT NULL, + ops JSONB + )", + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query("TRUNCATE bench_repo_seq") + .execute(&pool) + .await + .unwrap(); + + let events_per_task = event_count / concurrency; + let actual_count = events_per_task * concurrency; + + let start = Instant::now(); + + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = pool.clone(); + tokio::spawn(async move { + futures::stream::iter(0..events_per_task) + .then(|i| { + let pool = pool.clone(); + async move { + let global = task_id * events_per_task + i; + let did = format!("did:plc:{global:024x}"); + let ops_size = match global % 4 { + 0 => 64, + 1 => 256, + 2 => 1024, + _ => 4096, + }; + let payload: String = (0..ops_size) + .map(|j| { + ((global.wrapping_mul(31).wrapping_add(j)) % 26 + 97) as u8 + as char + }) + .collect(); + let ops = serde_json::json!({ "data": payload }); + let t = Instant::now(); + sqlx::query( + "INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)", + ) + .bind(&did) + .bind("commit") + .bind(&ops) + .execute(&pool) + .await + .unwrap(); + t.elapsed() + } + }) + .collect::>() + .await + }) + }) + .collect(); + + let mut all_latencies: Vec = futures::future::join_all(handles) + .await + .into_iter() + .flat_map(Result::unwrap) + .collect(); + let elapsed = start.elapsed(); + + let lat = format_latency(compute_stats(&mut all_latencies).as_ref()); + println!( + "{:.0} events/sec, {:.1}ms{lat}", + actual_count as f64 / elapsed.as_secs_f64(), + elapsed.as_secs_f64() * 1000.0, + ); + + sqlx::query("TRUNCATE bench_repo_seq") + .execute(&pool) + .await + .unwrap(); + pool.close().await; +} + +async fn bench_pg_read_throughput(event_count: usize, concurrency: usize) { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + println!("skipped, set DATABASE_URL to enable"); + return; + } + }; + + let max_conns = u32::try_from(concurrency) + .unwrap_or(u32::MAX) + .saturating_add(5); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(max_conns) + .connect(&database_url) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS bench_repo_seq ( + seq BIGSERIAL PRIMARY KEY, + did TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + event_type TEXT NOT NULL, + ops JSONB + )", + ) + .execute(&pool) + .await + .unwrap(); + + let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM bench_repo_seq") + .fetch_one(&pool) + .await + .unwrap(); + if (row.0 as usize) < event_count { + sqlx::query("TRUNCATE bench_repo_seq") + .execute(&pool) + .await + .unwrap(); + println!("populating {event_count} events"); + futures::stream::iter(0..event_count) + .map(|i| { + let pool = pool.clone(); + async move { + let did = format!("did:plc:{i:024x}"); + let ops = serde_json::json!({ "data": "x".repeat(256) }); + sqlx::query( + "INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)", + ) + .bind(&did) + .bind("commit") + .bind(&ops) + .execute(&pool) + .await + .unwrap(); + } + }) + .buffer_unordered(50) + .collect::>() + .await; + } + + let total_events = Arc::new(AtomicU64::new(0)); + + let start = Instant::now(); + + let handles: Vec<_> = (0..concurrency) + .map(|_| { + let pool = pool.clone(); + let total_events = Arc::clone(&total_events); + tokio::spawn(async move { + let mut cursor = 0i64; + let mut count = 0u64; + loop { + let rows: Vec<(i64,)> = sqlx::query_as( + "SELECT seq FROM bench_repo_seq WHERE seq > $1 ORDER BY seq LIMIT $2", + ) + .bind(cursor) + .bind(1000i64) + .fetch_all(&pool) + .await + .unwrap(); + if rows.is_empty() { + break; + } + count += rows.len() as u64; + cursor = rows.last().unwrap().0; + } + total_events.fetch_add(count, Ordering::Relaxed); + }) + }) + .collect(); + + futures::future::join_all(handles).await; + let elapsed = start.elapsed(); + + let total = total_events.load(Ordering::Relaxed); + println!( + "{:.0} total events/sec across {concurrency} readers, {total} events, {:.1}ms", + total as f64 / elapsed.as_secs_f64(), + elapsed.as_secs_f64() * 1000.0, + ); + + pool.close().await; +} + +fn main() { + println!("-- eventlog benchmarks --"); + let cpus = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8); + println!("available parallelism: {cpus}"); + + let parse_env_list = |var: &str, defaults: Vec| -> Vec { + std::env::var(var).map_or(defaults, |s| { + s.split(',') + .map(|n| { + n.trim() + .replace('_', "") + .parse::() + .unwrap_or_else(|e| panic!("{var}: {e}")) + }) + .collect() + }) + }; + + let event_counts = parse_env_list("BENCH_EVENT_COUNTS", vec![10_000, 100_000]); + let large_event_counts = parse_env_list("BENCH_LARGE_EVENT_COUNTS", vec![1_000_000]); + let producer_counts = parse_env_list("BENCH_PRODUCERS", vec![1, 10, 50, 100, 500]); + + let all_write_counts: Vec = event_counts + .iter() + .chain(large_event_counts.iter()) + .copied() + .collect(); + + println!("event counts: {event_counts:?}, large: {large_event_counts:?}"); + println!("producer counts: {producer_counts:?}"); + + println!("-- write throughput --"); + + all_write_counts.iter().for_each(|&n| { + bench_sequential_append(n); + }); + + all_write_counts.iter().for_each(|&n| { + producer_counts.iter().for_each(|&p| { + if n >= p { + bench_concurrent_producers(n, p); + } + }); + }); + + all_write_counts.iter().for_each(|&n| { + [256usize, 1024, 4096].iter().for_each(|&batch| { + bench_batch_append(n, batch); + }); + }); + + println!("-- rotation --"); + + event_counts.iter().for_each(|&n| { + bench_rotation_under_load(n); + }); + + println!("-- read throughput --"); + + event_counts.iter().for_each(|&n| { + bench_sequential_scan(n); + }); + + event_counts.iter().for_each(|&n| { + [2usize, 4, 8, 16, 32].iter().for_each(|&r| { + bench_parallel_readers(n, r); + }); + }); + + println!("-- broadcast fanout --"); + + [1usize, 10, 100, 500, 1000].iter().for_each(|&s| { + bench_broadcast_fanout(s); + }); + + println!("-- stampede --"); + + bench_stampede(100_000, 50, 8, 10); + bench_stampede(100_000, 100, 16, 50); + bench_stampede(500_000, 100, 16, 50); + + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(cpus) + .enable_all() + .build() + .unwrap(); + + if std::env::var("DATABASE_URL").is_ok() { + println!("-- postgres comparison --"); + + event_counts.iter().for_each(|&n| { + producer_counts.iter().for_each(|&p| { + if n >= p { + println!("-- postgres write: {n} events, {p} writers --",); + rt.block_on(bench_pg_write_throughput(n, p)); + } + }); + }); + + event_counts.iter().for_each(|&n| { + [1usize, 4, 16, 32].iter().for_each(|&r| { + println!("-- postgres read: {n} events, {r} readers --",); + rt.block_on(bench_pg_read_throughput(n, r)); + }); + }); + + rt.block_on(async { + let url = std::env::var("DATABASE_URL").unwrap(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .unwrap(); + sqlx::query("DROP TABLE IF EXISTS bench_repo_seq") + .execute(&pool) + .await + .unwrap(); + pool.close().await; + }); + } else { + println!("set DATABASE_URL for postgres comparison"); + } +} diff --git a/crates/tranquil-store/src/blockstore/group_commit.rs b/crates/tranquil-store/src/blockstore/group_commit.rs index 2553e08..50cf054 100644 --- a/crates/tranquil-store/src/blockstore/group_commit.rs +++ b/crates/tranquil-store/src/blockstore/group_commit.rs @@ -4,6 +4,9 @@ use std::io; use std::sync::Arc; use std::thread; +use crate::fsync_order::PostBlockstoreHook; + +use super::BlocksSynced; use crate::io::{FileId, OpenOptions, StorageIO}; use super::data_file::{CID_SIZE, DataFileWriter}; @@ -109,6 +112,15 @@ impl GroupCommitWriter { manager: DataFileManager, index: Arc, config: GroupCommitConfig, + ) -> Result { + Self::spawn_with_hook(manager, index, config, None) + } + + pub fn spawn_with_hook( + manager: DataFileManager, + index: Arc, + config: GroupCommitConfig, + post_sync_hook: Option>, ) -> Result { let cursor = index.read_write_cursor().map_err(CommitError::from)?; let mut state = initialize_active_state(&manager, cursor)?; @@ -118,7 +130,14 @@ impl GroupCommitWriter { let handle = thread::Builder::new() .name("blockstore-group-commit".into()) .spawn(move || { - commit_loop(&manager, &*index, &receiver, &config, &mut state); + commit_loop( + &manager, + &index, + &receiver, + &config, + &mut state, + post_sync_hook.as_deref(), + ); }) .map_err(|e| CommitError::from(io::Error::other(e)))?; @@ -284,6 +303,7 @@ fn commit_loop( receiver: &flume::Receiver, config: &GroupCommitConfig, state: &mut ActiveState, + post_sync_hook: Option<&dyn PostBlockstoreHook>, ) { loop { let first = match receiver.recv() { @@ -302,24 +322,37 @@ fn commit_loop( let result = process_batch(manager, index, &batch, state); + if let Ok((ref _dedup, ref proof)) = result { + run_post_sync_hook(post_sync_hook, proof); + } + if let Err(ref e) = result { tracing::warn!(error = %e, "commit batch failed"); } - dispatch_responses(batch, result); + dispatch_responses(batch, result.map(|(dedup, _proof)| dedup)); if shutdown_after { - drain_and_process_remaining(manager, index, receiver, state); + drain_and_process_remaining(manager, index, receiver, state, post_sync_hook); return; } } } +fn run_post_sync_hook(hook: Option<&dyn PostBlockstoreHook>, proof: &BlocksSynced) { + if let Some(hook) = hook + && let Err(e) = hook.on_blocks_synced(proof) + { + tracing::error!(error = %e, "post-blockstore sync hook failed"); + } +} + fn drain_and_process_remaining( manager: &DataFileManager, index: &KeyIndex, receiver: &flume::Receiver, state: &mut ActiveState, + post_sync_hook: Option<&dyn PostBlockstoreHook>, ) { let entries: Vec = std::iter::from_fn(|| receiver.try_recv().ok()) .filter_map(|req| classify_request(req).ok()) @@ -330,7 +363,12 @@ fn drain_and_process_remaining( } let result = process_batch(manager, index, &entries, state); - dispatch_responses(entries, result); + + if let Ok((ref _dedup, ref proof)) = result { + run_post_sync_hook(post_sync_hook, proof); + } + + dispatch_responses(entries, result.map(|(dedup, _proof)| dedup)); } struct RotationState { @@ -343,7 +381,7 @@ fn process_batch( index: &KeyIndex, batch: &[BatchEntry], state: &mut ActiveState, -) -> Result, CommitError> { +) -> Result<(HashMap<[u8; CID_SIZE], BlockLocation>, BlocksSynced), CommitError> { let mut dedup: HashMap<[u8; CID_SIZE], BlockLocation> = HashMap::new(); let mut index_entries: Vec<([u8; CID_SIZE], BlockLocation)> = Vec::new(); let mut all_decrements: Vec<[u8; CID_SIZE]> = Vec::new(); @@ -446,7 +484,7 @@ fn process_batch( .batch_put(&index_entries, &all_decrements, cursor) .map_err(CommitError::from)?; - Ok(dedup) + Ok((dedup, BlocksSynced::new())) } fn dispatch_responses( diff --git a/crates/tranquil-store/src/blockstore/mod.rs b/crates/tranquil-store/src/blockstore/mod.rs index c56742e..a1b918f 100644 --- a/crates/tranquil-store/src/blockstore/mod.rs +++ b/crates/tranquil-store/src/blockstore/mod.rs @@ -32,6 +32,14 @@ use std::path::Path; use crate::io::StorageIO; +pub struct BlocksSynced(()); + +impl BlocksSynced { + pub(in crate::blockstore) fn new() -> Self { + Self(()) + } +} + pub(crate) fn list_files_by_extension( io: &S, dir: &Path, diff --git a/crates/tranquil-store/src/blockstore/store.rs b/crates/tranquil-store/src/blockstore/store.rs index aaea967..cdc294b 100644 --- a/crates/tranquil-store/src/blockstore/store.rs +++ b/crates/tranquil-store/src/blockstore/store.rs @@ -10,6 +10,7 @@ use jacquard_repo::storage::BlockStore; use multihash::Multihash; use sha2::{Digest, Sha256}; +use crate::fsync_order::PostBlockstoreHook; use crate::io::{OpenOptions, RealIO, StorageIO}; use super::data_file::{BLOCK_RECORD_OVERHEAD, CID_SIZE, ReadBlockRecord}; @@ -101,6 +102,13 @@ impl Drop for WriterHandle { impl TranquilBlockStore { pub fn open(config: BlockStoreConfig) -> Result { + Self::open_with_hook(config, None) + } + + pub fn open_with_hook( + config: BlockStoreConfig, + post_sync_hook: Option>, + ) -> Result { if config.data_dir == config.index_dir { return Err(RepoError::storage(io::Error::new( io::ErrorKind::InvalidInput, @@ -126,9 +134,13 @@ impl TranquilBlockStore { let manager_for_writer = DataFileManager::new(RealIO::new(), config.data_dir.clone(), config.max_file_size); - let writer = - GroupCommitWriter::spawn(manager_for_writer, Arc::clone(&index), config.group_commit) - .map_err(commit_error_to_repo)?; + let writer = GroupCommitWriter::spawn_with_hook( + manager_for_writer, + Arc::clone(&index), + config.group_commit, + post_sync_hook, + ) + .map_err(commit_error_to_repo)?; let sender = writer.sender().clone(); let manager_for_reader = Arc::new(DataFileManager::new( diff --git a/crates/tranquil-store/src/eventlog/bridge.rs b/crates/tranquil-store/src/eventlog/bridge.rs new file mode 100644 index 0000000..2712490 --- /dev/null +++ b/crates/tranquil-store/src/eventlog/bridge.rs @@ -0,0 +1,296 @@ +use std::io; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use tracing::warn; +use tranquil_db_traits::{DbError, SequenceNumber, SequencedEvent}; + +use super::notifier::EventLogNotifier; +use super::types::{EventSequence, TimestampMicros}; +use super::writer::SyncResult; +use super::{EventLog, EventWithMutations, decode_payload, to_sequenced_event}; +use crate::io::StorageIO; + +pub struct DeferredBroadcast(SyncResult); + +fn io_to_db(e: io::Error) -> DbError { + DbError::Query(e.to_string()) +} + +fn seq_to_event(seq: SequenceNumber) -> EventSequence { + let raw = seq.as_i64(); + if raw < 0 { + warn!( + seq = raw, + "negative SequenceNumber passed to eventlog bridge, treating as BEFORE_ALL" + ); + return EventSequence::BEFORE_ALL; + } + EventSequence::cursor_from_i64(raw).unwrap_or(EventSequence::BEFORE_ALL) +} + +fn datetime_to_micros(dt: &DateTime) -> u64 { + let micros = dt.timestamp_micros(); + debug_assert!(micros >= 0, "pre-epoch DateTime passed to eventlog bridge"); + u64::try_from(micros).unwrap_or(0) +} + +pub struct EventLogBridge { + log: Arc>, +} + +impl EventLogBridge { + pub fn new(log: Arc>) -> Self { + Self { log } + } + + pub fn notifier(&self) -> EventLogNotifier { + EventLogNotifier::new(Arc::clone(&self.log)) + } + + pub fn log(&self) -> &Arc> { + &self.log + } + + pub fn get_max_seq(&self) -> SequenceNumber { + let es = self.log.max_seq(); + SequenceNumber::from_raw(es.as_i64()) + } + + pub fn get_events_since_seq( + &self, + since: SequenceNumber, + limit: Option, + ) -> Result, DbError> { + let cap = limit + .and_then(|l| usize::try_from(l).ok()) + .unwrap_or(usize::MAX); + self.get_events_impl(since, cap) + } + + pub fn get_events_since_cursor( + &self, + cursor: SequenceNumber, + limit: i64, + ) -> Result, DbError> { + let cap = usize::try_from(limit).unwrap_or(usize::MAX); + self.get_events_impl(cursor, cap) + } + + fn get_events_impl( + &self, + since: SequenceNumber, + limit: usize, + ) -> Result, DbError> { + let cursor = seq_to_event(since); + self.log.get_events_since(cursor, limit).map_err(io_to_db) + } + + pub fn get_event_by_seq(&self, seq: SequenceNumber) -> Result, DbError> { + let es = EventSequence::from_i64(seq.as_i64()) + .ok_or_else(|| DbError::Query("invalid sequence number".into()))?; + self.log.get_event(es).map_err(io_to_db) + } + + pub fn get_events_in_seq_range( + &self, + start: SequenceNumber, + end: SequenceNumber, + ) -> Result, DbError> { + let end_raw = match u64::try_from(end.as_i64()) { + Ok(v) => v, + Err(_) => return Ok(Vec::new()), + }; + let cursor = seq_to_event(start); + if end_raw <= cursor.raw().saturating_add(1) { + return Ok(Vec::new()); + } + let range_size = + usize::try_from(end_raw.saturating_sub(cursor.raw())).unwrap_or(usize::MAX); + let raw_events = self + .log + .reader() + .read_events_from(cursor, range_size) + .map_err(io_to_db)?; + + raw_events + .iter() + .take_while(|e| e.seq.raw() < end_raw) + .map(|raw| { + let payload = + decode_payload(&raw.payload).map_err(|e| DbError::Query(e.to_string()))?; + to_sequenced_event(raw, &payload).map_err(|e| DbError::Query(e.to_string())) + }) + .collect() + } + + pub fn get_min_seq_since( + &self, + since: DateTime, + ) -> Result, DbError> { + let target_us = datetime_to_micros(&since); + let target_ts = TimestampMicros::new(target_us); + let reader = self.log.reader(); + + let segments = self.log.manager().list_segments().map_err(io_to_db)?; + if segments.is_empty() { + return Ok(None); + } + + let scan_from_seg = self.find_segment_for_timestamp(&segments, target_ts)?; + + let start_seq = match scan_from_seg { + Some(idx) => reader + .load_index(segments[idx]) + .map_err(io_to_db)? + .first_seq() + .map(|s| s.prev_or_before_all()) + .unwrap_or(EventSequence::BEFORE_ALL), + None => return Ok(None), + }; + + const SCAN_BATCH: usize = 1024; + self.scan_for_timestamp(reader, start_seq, target_ts, SCAN_BATCH) + } + + fn scan_for_timestamp( + &self, + reader: &super::EventLogReader, + cursor: EventSequence, + target_ts: TimestampMicros, + batch_size: usize, + ) -> Result, DbError> { + let batch = reader + .read_events_from(cursor, batch_size) + .map_err(io_to_db)?; + if batch.is_empty() { + return Ok(None); + } + match batch.iter().find(|e| e.timestamp >= target_ts) { + Some(e) => Ok(Some(SequenceNumber::from_raw(e.seq.as_i64()))), + None => { + let next_cursor = batch.last().map(|e| e.seq).unwrap_or(cursor); + self.scan_for_timestamp(reader, next_cursor, target_ts, batch_size) + } + } + } + + fn find_segment_for_timestamp( + &self, + segments: &[super::SegmentId], + target_ts: TimestampMicros, + ) -> Result, DbError> { + let reader = self.log.reader(); + + let last_seg_idx = segments.len() - 1; + let last_index = reader + .load_index(segments[last_seg_idx]) + .map_err(io_to_db)?; + + let last_ts = last_index + .first_seq() + .and_then(|seq| reader.read_event_at(seq).ok().flatten()) + .map(|e| e.timestamp); + + match last_ts { + Some(ts) if ts < target_ts => { + let tail_ts = last_index + .last_seq() + .and_then(|seq| reader.read_event_at(seq).ok().flatten()) + .map(|e| e.timestamp); + match tail_ts { + Some(ts) if ts < target_ts => return Ok(None), + _ => return Ok(Some(last_seg_idx)), + } + } + None => return Ok(None), + _ => {} + } + + Ok(self + .binary_search_segment(reader, segments, target_ts, 0, last_seg_idx, None)? + .map(|r| r.saturating_sub(1))) + } + + fn binary_search_segment( + &self, + reader: &super::EventLogReader, + segments: &[super::SegmentId], + target_ts: TimestampMicros, + lo: usize, + hi: usize, + best: Option, + ) -> Result, DbError> { + if lo > hi { + return Ok(best); + } + let mid = lo + (hi - lo) / 2; + let seg_ts = reader + .load_index(segments[mid]) + .map_err(io_to_db)? + .first_seq() + .and_then(|seq| reader.read_event_at(seq).ok().flatten()) + .map(|e| e.timestamp); + + match seg_ts { + Some(ts) if ts < target_ts => { + self.binary_search_segment(reader, segments, target_ts, mid + 1, hi, best) + } + Some(_) => match mid { + 0 => Ok(Some(0)), + _ => { + self.binary_search_segment(reader, segments, target_ts, lo, mid - 1, Some(mid)) + } + }, + None => self.binary_search_segment(reader, segments, target_ts, mid + 1, hi, best), + } + } + + pub fn get_events_with_mutations_since( + &self, + since: SequenceNumber, + limit: usize, + ) -> Result, DbError> { + let cursor = seq_to_event(since); + self.log + .get_events_with_mutations_since(cursor, limit) + .map_err(io_to_db) + } + + pub fn insert_event(&self, event: &SequencedEvent) -> Result { + let seq = self + .log + .append_and_sync(&event.did, event.event_type, event)?; + Ok(SequenceNumber::from_raw(seq.as_i64())) + } + + pub fn insert_event_deferred( + &self, + event: &SequencedEvent, + ) -> Result<(SequenceNumber, DeferredBroadcast), io::Error> { + let seq = self.log.append_event(&event.did, event.event_type, event)?; + let sync_result = self.log.sync_data()?; + Ok(( + SequenceNumber::from_raw(seq.as_i64()), + DeferredBroadcast(sync_result), + )) + } + + pub fn insert_event_deferred_raw( + &self, + did: &tranquil_types::Did, + event_type: tranquil_db_traits::RepoEventType, + payload: Vec, + ) -> Result<(SequenceNumber, DeferredBroadcast), io::Error> { + let seq = self.log.append_raw_payload(did, event_type, payload)?; + let sync_result = self.log.sync_data()?; + Ok(( + SequenceNumber::from_raw(seq.as_i64()), + DeferredBroadcast(sync_result), + )) + } + + pub fn complete_broadcast(&self, deferred: DeferredBroadcast) { + self.log.broadcast_result(&deferred.0); + } +} diff --git a/crates/tranquil-store/src/eventlog/manager.rs b/crates/tranquil-store/src/eventlog/manager.rs new file mode 100644 index 0000000..5abb833 --- /dev/null +++ b/crates/tranquil-store/src/eventlog/manager.rs @@ -0,0 +1,659 @@ +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use parking_lot::RwLock; + +use crate::io::{FileId, OpenOptions, StorageIO}; + +use super::segment_file::SEGMENT_HEADER_SIZE; +use super::segment_index::SegmentIndex; +use super::types::{SegmentId, SegmentOffset}; + +pub(crate) const SEGMENT_FILE_EXTENSION: &str = "tqe"; +pub(crate) const INDEX_FILE_EXTENSION: &str = "tqi"; + +struct CachedSegmentHandle { + fd: FileId, + sealed: bool, + writable: bool, +} + +pub struct SegmentManager { + io: S, + segments_dir: PathBuf, + max_segment_size: u64, + handles: RwLock>, + retention_epoch: AtomicU64, +} + +impl SegmentManager { + pub fn new(io: S, segments_dir: PathBuf, max_segment_size: u64) -> io::Result { + assert!( + max_segment_size > SEGMENT_HEADER_SIZE as u64, + "max_segment_size ({max_segment_size}) must exceed SEGMENT_HEADER_SIZE ({SEGMENT_HEADER_SIZE})" + ); + io.mkdir(&segments_dir)?; + Ok(Self { + io, + segments_dir, + max_segment_size, + handles: RwLock::new(HashMap::new()), + retention_epoch: AtomicU64::new(0), + }) + } + + pub fn io(&self) -> &S { + &self.io + } + + pub fn segments_dir(&self) -> &Path { + &self.segments_dir + } + + pub fn max_segment_size(&self) -> u64 { + self.max_segment_size + } + + pub fn segment_path(&self, id: SegmentId) -> PathBuf { + self.segments_dir + .join(format!("{id}.{SEGMENT_FILE_EXTENSION}")) + } + + pub fn index_path(&self, id: SegmentId) -> PathBuf { + self.segments_dir + .join(format!("{id}.{INDEX_FILE_EXTENSION}")) + } + + pub fn list_segments(&self) -> io::Result> { + let entries = self.io.list_dir(&self.segments_dir)?; + let mut ids: Vec = entries + .iter() + .filter_map(|path| { + let stem = path.file_stem()?.to_str()?; + let ext = path.extension()?.to_str()?; + (ext == SEGMENT_FILE_EXTENSION) + .then(|| stem.parse::().ok().map(SegmentId::new))? + }) + .collect(); + ids.sort(); + Ok(ids) + } + + pub fn open_for_read(&self, id: SegmentId) -> io::Result { + if let Some(entry) = self.handles.read().get(&id) { + return Ok(entry.fd); + } + let path = self.segment_path(id); + let fd = self.io.open(&path, OpenOptions::read_only_existing())?; + let mut cache = self.handles.write(); + match cache.get(&id) { + Some(entry) => { + let _ = self.io.close(fd); + Ok(entry.fd) + } + None => { + cache.insert( + id, + CachedSegmentHandle { + fd, + sealed: false, + writable: false, + }, + ); + Ok(fd) + } + } + } + + pub fn open_for_append(&self, id: SegmentId) -> io::Result { + { + let cache = self.handles.read(); + if let Some(entry) = cache.get(&id) { + if entry.sealed { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("cannot append to sealed segment {id}"), + )); + } + if entry.writable { + return Ok(entry.fd); + } + } + } + let path = self.segment_path(id); + let fd = self.io.open(&path, OpenOptions::read_write())?; + let mut cache = self.handles.write(); + match cache.get(&id) { + Some(entry) if entry.sealed => { + let _ = self.io.close(fd); + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("cannot append to sealed segment {id}"), + )) + } + Some(entry) if entry.writable => { + let _ = self.io.close(fd); + Ok(entry.fd) + } + Some(entry) => { + let old_fd = entry.fd; + cache.insert( + id, + CachedSegmentHandle { + fd, + sealed: false, + writable: true, + }, + ); + let _ = self.io.close(old_fd); + Ok(fd) + } + None => { + cache.insert( + id, + CachedSegmentHandle { + fd, + sealed: false, + writable: true, + }, + ); + Ok(fd) + } + } + } + + pub fn should_rotate(&self, position: SegmentOffset) -> bool { + position.raw() >= self.max_segment_size + } + + pub fn prepare_rotation(&self, current_id: SegmentId) -> io::Result<(SegmentId, FileId)> { + let next = current_id.next(); + let path = self.segment_path(next); + let fd = self.io.open(&path, OpenOptions::read_write())?; + self.io.truncate(fd, 0)?; + self.io.sync_dir(&self.segments_dir)?; + Ok((next, fd)) + } + + pub fn commit_rotation(&self, new_id: SegmentId, fd: FileId) { + self.handles.write().insert( + new_id, + CachedSegmentHandle { + fd, + sealed: false, + writable: true, + }, + ); + } + + pub fn seal_segment(&self, id: SegmentId, index: &SegmentIndex) -> io::Result<()> { + let path = self.index_path(id); + index.save(&self.io, &path)?; + let mut cache = self.handles.write(); + let entry = cache.get_mut(&id).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("seal_segment: segment {id} not in handle cache"), + ) + })?; + entry.sealed = true; + Ok(()) + } + + pub fn is_sealed(&self, id: SegmentId) -> bool { + self.handles + .read() + .get(&id) + .is_some_and(|entry| entry.sealed) + } + + pub fn rollback_rotation(&self, new_id: SegmentId, fd: FileId) { + let _ = self.io.close(fd); + self.handles.write().remove(&new_id); + let _ = self.io.delete(&self.segment_path(new_id)); + } + + pub fn delete_segment(&self, id: SegmentId) -> io::Result<()> { + { + let mut cache = self.handles.write(); + if let Some(entry) = cache.remove(&id) { + let _ = self.io.close(entry.fd); + } + } + match self.io.delete(&self.index_path(id)) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + self.io.delete(&self.segment_path(id))?; + self.io.sync_dir(&self.segments_dir)?; + self.retention_epoch.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + + pub fn oldest_segment(&self) -> io::Result> { + self.list_segments().map(|segs| segs.into_iter().next()) + } + + pub fn retention_epoch(&self) -> u64 { + self.retention_epoch.load(Ordering::Relaxed) + } + + pub fn shutdown(&self) { + self.handles.write().drain().for_each(|(_, handle)| { + let _ = self.io.close(handle.fd); + }); + } +} + +impl Drop for SegmentManager { + fn drop(&mut self) { + self.shutdown(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::segment_file::{SegmentWriter, ValidEvent}; + use crate::eventlog::segment_index::{DEFAULT_INDEX_INTERVAL, rebuild_from_segment}; + use crate::eventlog::types::{ + DidHash, EventSequence, EventTypeTag, SegmentOffset, TimestampMicros, + }; + use crate::sim::SimulatedIO; + + fn setup_manager(max_segment_size: u64) -> SegmentManager { + let sim = SimulatedIO::pristine(42); + SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap() + } + + fn test_event(seq: u64, payload: &[u8]) -> ValidEvent { + ValidEvent { + seq: EventSequence::new(seq), + timestamp: TimestampMicros::new(seq * 1_000_000), + did_hash: DidHash::from_did(&format!("did:plc:test{seq}")), + event_type: EventTypeTag::COMMIT, + payload: payload.to_vec(), + } + } + + #[test] + fn new_creates_directory() { + let sim = SimulatedIO::pristine(42); + let mgr = SegmentManager::new(sim, PathBuf::from("/eventlog/segments"), 1024).unwrap(); + let entries = mgr.io().list_dir(Path::new("/eventlog/segments")).unwrap(); + assert!(entries.is_empty()); + } + + #[test] + fn segment_path_format() { + let mgr = setup_manager(1024); + assert_eq!( + mgr.segment_path(SegmentId::new(0)), + Path::new("/segments/00000000.tqe") + ); + assert_eq!( + mgr.segment_path(SegmentId::new(42)), + Path::new("/segments/00000042.tqe") + ); + } + + #[test] + fn index_path_format() { + let mgr = setup_manager(1024); + assert_eq!( + mgr.index_path(SegmentId::new(0)), + Path::new("/segments/00000000.tqi") + ); + assert_eq!( + mgr.index_path(SegmentId::new(7)), + Path::new("/segments/00000007.tqi") + ); + } + + #[test] + fn open_for_append_creates_file() { + let mgr = setup_manager(1024); + let fd = mgr.open_for_append(SegmentId::new(1)).unwrap(); + assert_eq!(mgr.io().file_size(fd).unwrap(), 0); + } + + #[test] + fn open_for_read_missing_file_errors() { + let mgr = setup_manager(1024); + assert!(mgr.open_for_read(SegmentId::new(99)).is_err()); + } + + #[test] + fn handle_cache_returns_same_fd() { + let mgr = setup_manager(1024); + let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap(); + assert_eq!(fd1, fd2); + } + + #[test] + fn open_for_read_uses_cache_from_append() { + let mgr = setup_manager(1024); + let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap(); + assert_eq!(fd_write, fd_read); + } + + #[test] + fn list_segments_finds_segment_files() { + let mgr = setup_manager(1024); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.open_for_append(SegmentId::new(3)).unwrap(); + + let segments = mgr.list_segments().unwrap(); + assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(3)]); + } + + #[test] + fn list_segments_ignores_non_segment_files() { + let mgr = setup_manager(1024); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.io() + .open(Path::new("/segments/notes.txt"), OpenOptions::read_write()) + .unwrap(); + + let segments = mgr.list_segments().unwrap(); + assert_eq!(segments, vec![SegmentId::new(1)]); + } + + #[test] + fn list_segments_ignores_index_files() { + let mgr = setup_manager(1024); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.io() + .open( + Path::new("/segments/00000001.tqi"), + OpenOptions::read_write(), + ) + .unwrap(); + + let segments = mgr.list_segments().unwrap(); + assert_eq!(segments, vec![SegmentId::new(1)]); + } + + #[test] + fn list_segments_sorted_ascending() { + let mgr = setup_manager(1024); + mgr.open_for_append(SegmentId::new(5)).unwrap(); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.open_for_append(SegmentId::new(3)).unwrap(); + + let segments = mgr.list_segments().unwrap(); + assert_eq!( + segments, + vec![SegmentId::new(1), SegmentId::new(3), SegmentId::new(5)] + ); + } + + #[test] + fn should_rotate_respects_threshold() { + let mgr = setup_manager(1024); + assert!(!mgr.should_rotate(SegmentOffset::new(100))); + assert!(!mgr.should_rotate(SegmentOffset::new(1023))); + assert!(mgr.should_rotate(SegmentOffset::new(1024))); + assert!(mgr.should_rotate(SegmentOffset::new(2000))); + } + + #[test] + fn rotation_lifecycle_prepare_commit() { + let mgr = setup_manager(1024); + let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap(); + assert_eq!(next_id, SegmentId::new(2)); + assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0); + mgr.commit_rotation(next_id, next_fd); + assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd); + } + + #[test] + fn rotation_rollback_cleans_up() { + let mgr = setup_manager(1024); + let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap(); + mgr.commit_rotation(next_id, next_fd); + + assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd); + mgr.rollback_rotation(next_id, next_fd); + + let segments = mgr.list_segments().unwrap(); + assert_eq!(segments, vec![SegmentId::new(1)]); + } + + #[test] + fn seal_segment_persists_index_and_marks_sealed() { + let mgr = setup_manager(64 * 1024); + let fd = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let mut writer = + SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + (1u64..=10).for_each(|i| { + writer + .append_event(mgr.io(), &test_event(i, format!("payload-{i}").as_bytes())) + .unwrap(); + }); + writer.sync(mgr.io()).unwrap(); + + let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap(); + + assert!(!mgr.is_sealed(SegmentId::new(1))); + mgr.seal_segment(SegmentId::new(1), &index).unwrap(); + assert!(mgr.is_sealed(SegmentId::new(1))); + + let loaded = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1))) + .unwrap() + .unwrap(); + assert_eq!(loaded, index); + } + + #[test] + fn delete_segment_removes_files_and_handle() { + let mgr = setup_manager(64 * 1024); + let fd = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let mut writer = + SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer + .append_event(mgr.io(), &test_event(1, b"will be deleted")) + .unwrap(); + writer.sync(mgr.io()).unwrap(); + + let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap(); + mgr.seal_segment(SegmentId::new(1), &index).unwrap(); + + let epoch_before = mgr.retention_epoch(); + mgr.delete_segment(SegmentId::new(1)).unwrap(); + assert_eq!(mgr.retention_epoch(), epoch_before + 1); + + assert!(mgr.list_segments().unwrap().is_empty()); + assert!(mgr.open_for_read(SegmentId::new(1)).is_err()); + } + + #[test] + fn oldest_segment_returns_first() { + let mgr = setup_manager(1024); + assert_eq!(mgr.oldest_segment().unwrap(), None); + + mgr.open_for_append(SegmentId::new(3)).unwrap(); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.open_for_append(SegmentId::new(5)).unwrap(); + + assert_eq!(mgr.oldest_segment().unwrap(), Some(SegmentId::new(1))); + } + + #[test] + fn retention_epoch_starts_at_zero() { + let mgr = setup_manager(1024); + assert_eq!(mgr.retention_epoch(), 0); + } + + #[test] + fn rotate_and_write_across_segments() { + let mgr = setup_manager(1024); + + let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let mut writer1 = + SegmentWriter::new(mgr.io(), fd1, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer1 + .append_event(mgr.io(), &test_event(1, b"first segment")) + .unwrap(); + writer1.sync(mgr.io()).unwrap(); + + let (id2, fd2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap(); + mgr.commit_rotation(id2, fd2); + + let mut writer2 = SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2)).unwrap(); + writer2 + .append_event(mgr.io(), &test_event(2, b"second segment")) + .unwrap(); + writer2.sync(mgr.io()).unwrap(); + + let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap(); + let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!(events1.len(), 1); + assert_eq!(events1[0].payload, b"first segment"); + + let fd2_read = mgr.open_for_read(id2).unwrap(); + let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!(events2.len(), 1); + assert_eq!(events2[0].payload, b"second segment"); + } + + #[test] + fn seal_then_append_errors() { + let mgr = setup_manager(64 * 1024); + let fd = mgr.open_for_append(SegmentId::new(1)).unwrap(); + SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let index = SegmentIndex::new(); + mgr.seal_segment(SegmentId::new(1), &index).unwrap(); + + let result = mgr.open_for_append(SegmentId::new(1)); + assert!(result.is_err()); + } + + #[test] + fn accessors() { + let mgr = setup_manager(999); + assert_eq!(mgr.max_segment_size(), 999); + assert_eq!(mgr.segments_dir(), Path::new("/segments")); + } + + #[test] + fn multiple_deletions_increment_epoch() { + let mgr = setup_manager(1024); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.open_for_append(SegmentId::new(2)).unwrap(); + mgr.open_for_append(SegmentId::new(3)).unwrap(); + + assert_eq!(mgr.retention_epoch(), 0); + mgr.delete_segment(SegmentId::new(1)).unwrap(); + assert_eq!(mgr.retention_epoch(), 1); + mgr.delete_segment(SegmentId::new(2)).unwrap(); + assert_eq!(mgr.retention_epoch(), 2); + } + + #[test] + fn open_for_read_does_not_infer_sealed_from_index_file() { + let mgr = setup_manager(64 * 1024); + let fd = mgr.open_for_append(SegmentId::new(1)).unwrap(); + let mut writer = + SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer + .append_event(mgr.io(), &test_event(1, b"sealed test")) + .unwrap(); + writer.sync(mgr.io()).unwrap(); + + let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap(); + mgr.seal_segment(SegmentId::new(1), &index).unwrap(); + + mgr.handles.write().remove(&SegmentId::new(1)); + + let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + assert!(!mgr.is_sealed(SegmentId::new(1))); + } + + #[test] + fn open_for_read_unsealed_allows_append() { + let mgr = setup_manager(1024); + let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap(); + + mgr.handles.write().remove(&SegmentId::new(1)); + + let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + assert!(!mgr.is_sealed(SegmentId::new(1))); + } + + #[test] + fn shutdown_clears_handles() { + let mgr = setup_manager(1024); + mgr.open_for_append(SegmentId::new(1)).unwrap(); + mgr.open_for_append(SegmentId::new(2)).unwrap(); + + mgr.shutdown(); + assert!(mgr.handles.read().is_empty()); + } + + #[test] + #[should_panic(expected = "max_segment_size")] + fn rejects_max_segment_size_too_small() { + let sim = SimulatedIO::pristine(42); + let _ = SegmentManager::new(sim, PathBuf::from("/segments"), 5); + } + + #[test] + fn prepare_rotation_truncates_stale_file() { + let mgr = setup_manager(1024); + let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap(); + + let stale_path = mgr.segment_path(SegmentId::new(2)); + let stale_fd = mgr + .io() + .open(&stale_path, OpenOptions::read_write()) + .unwrap(); + mgr.io().write_all_at(stale_fd, 0, &[0xDE; 4096]).unwrap(); + mgr.io().sync(stale_fd).unwrap(); + assert_eq!(mgr.io().file_size(stale_fd).unwrap(), 4096); + mgr.io().close(stale_fd).unwrap(); + + let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap(); + assert_eq!(next_id, SegmentId::new(2)); + assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0); + } + + #[test] + fn open_for_append_upgrades_read_only_handle() { + let mgr = setup_manager(1024); + let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap(); + + mgr.handles.write().remove(&SegmentId::new(1)); + + let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap(); + assert_ne!(fd_read, fd_append); + assert!(!mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable); + + let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap(); + assert_ne!(fd_upgraded, fd_read); + assert!(mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable); + } + + #[test] + fn seal_uncached_segment_returns_error() { + let mgr = setup_manager(1024); + let index = SegmentIndex::new(); + let result = mgr.seal_segment(SegmentId::new(99), &index); + assert!(result.is_err()); + } +} diff --git a/crates/tranquil-store/src/eventlog/mod.rs b/crates/tranquil-store/src/eventlog/mod.rs new file mode 100644 index 0000000..0a5b014 --- /dev/null +++ b/crates/tranquil-store/src/eventlog/mod.rs @@ -0,0 +1,489 @@ +mod bridge; +mod manager; +mod notifier; +mod payload; +mod reader; +mod segment_file; +mod segment_index; +mod types; +mod writer; + +use std::collections::VecDeque; +use std::io; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use tokio::sync::broadcast; +use tracing::warn; +use tranquil_db_traits::{RepoEventType, SequencedEvent}; +use tranquil_types::Did; + +use crate::blockstore::BlocksSynced; +use crate::fsync_order::PostBlockstoreHook; +use crate::io::StorageIO; + +pub use bridge::{DeferredBroadcast, EventLogBridge}; +pub use manager::SegmentManager; +pub use notifier::EventLogNotifier; +pub use payload::{ + EventPayload, PayloadError, decode_payload, encode_payload, encode_payload_with_mutations, + to_sequenced_event, validate_payload_size, +}; +pub use reader::{EventLogReader, RawEvent}; +pub use segment_file::{ + EVENT_HEADER_SIZE, EVENT_RECORD_OVERHEAD, ReadEventRecord, SEGMENT_FORMAT_VERSION, + SEGMENT_HEADER_SIZE, SEGMENT_MAGIC, SegmentReader, SegmentWriter, ValidEvent, + ValidateEventRecord, decode_event_record, encode_event_record, validate_event_record, +}; +pub use segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment}; +pub use types::{ + DEFAULT_SEGMENT_SIZE, DidHash, EventLength, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, + SegmentId, SegmentOffset, TimestampMicros, +}; +pub use writer::{EventLogWriter, SyncResult}; + +const DEFAULT_BROADCAST_BUFFER: usize = 16384; + +pub struct EventWithMutations { + pub event: SequencedEvent, + pub mutation_set: Option>, +} + +pub struct EventLogConfig { + pub segments_dir: PathBuf, + pub max_segment_size: u64, + pub index_interval: usize, + pub broadcast_buffer: usize, + pub use_mmap: bool, +} + +impl Default for EventLogConfig { + fn default() -> Self { + Self { + segments_dir: PathBuf::from("eventlog"), + max_segment_size: DEFAULT_SEGMENT_SIZE, + index_interval: DEFAULT_INDEX_INTERVAL, + broadcast_buffer: DEFAULT_BROADCAST_BUFFER, + use_mmap: true, + } + } +} + +pub struct EventLog { + writer: Mutex>, + reader: Arc>, + manager: Arc>, + broadcast_tx: broadcast::Sender, + synced_seq: AtomicU64, + consecutive_sync_failures: AtomicU32, +} + +impl EventLog { + pub fn open(config: EventLogConfig, io: S) -> io::Result { + let manager = Arc::new(SegmentManager::new( + io, + config.segments_dir, + config.max_segment_size, + )?); + + let writer = EventLogWriter::open(Arc::clone(&manager), config.index_interval)?; + let synced = writer.synced_seq(); + + let reader = Arc::new(EventLogReader::new(Arc::clone(&manager), config.use_mmap)); + reader.set_active_segment(writer.active_segment_id()); + reader.seed_index(writer.active_segment_id(), writer.active_index_snapshot()); + reader.refresh_segment_ranges()?; + + let (broadcast_tx, _) = broadcast::channel(config.broadcast_buffer); + + Ok(Self { + writer: Mutex::new(writer), + reader, + manager, + broadcast_tx, + synced_seq: AtomicU64::new(synced.raw()), + consecutive_sync_failures: AtomicU32::new(0), + }) + } + + pub fn append_event( + &self, + did: &Did, + event_type: RepoEventType, + event: &SequencedEvent, + ) -> io::Result { + let payload = encode_payload(event); + self.append_raw_payload(did, event_type, payload) + } + + pub fn append_raw_payload( + &self, + did: &Did, + event_type: RepoEventType, + payload: Vec, + ) -> io::Result { + let did_hash = DidHash::from_did(did.as_str()); + let tag = repo_event_type_to_tag(event_type); + validate_payload_size(&payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + self.writer.lock().append(did_hash, tag, payload) + } + + pub fn sync(&self) -> io::Result { + self.sync_and_broadcast() + } + + pub fn append_and_sync( + &self, + did: &Did, + event_type: RepoEventType, + event: &SequencedEvent, + ) -> io::Result { + let seq = self.append_event(did, event_type, event)?; + self.sync_and_broadcast()?; + Ok(seq) + } + + pub fn append_batch( + &self, + events: Vec<(&Did, RepoEventType, &SequencedEvent)>, + ) -> io::Result> { + let mut writer = self.writer.lock(); + events + .iter() + .map(|(did, event_type, event)| { + let did_hash = DidHash::from_did(did.as_str()); + let tag = repo_event_type_to_tag(*event_type); + let payload = encode_payload(event); + validate_payload_size(&payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + writer.append(did_hash, tag, payload) + }) + .collect() + } + + pub fn sync_data(&self) -> io::Result { + let mut writer = self.writer.lock(); + let result = writer.sync()?; + self.synced_seq + .store(result.synced_through.raw(), Ordering::Release); + + if let (Some(first), Some(last)) = + (result.flushed_events.first(), result.flushed_events.last()) + { + self.reader.extend_active_range(first.seq, last.seq); + } + Ok(result) + } + + pub fn broadcast_result(&self, result: &SyncResult) { + result.flushed_events.iter().for_each(|e| { + let _ = self.broadcast_tx.send(valid_event_to_raw(e)); + }); + } + + pub fn sync_and_broadcast(&self) -> io::Result { + let result = self.sync_data()?; + self.broadcast_result(&result); + Ok(result) + } + + pub fn get_events_since( + &self, + cursor: EventSequence, + limit: usize, + ) -> io::Result> { + let raw_events = self.reader.read_events_from(cursor, limit)?; + raw_events + .iter() + .map(|raw| { + let payload = decode_payload(&raw.payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + to_sequenced_event(raw, &payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + }) + .collect() + } + + pub fn get_events_with_mutations_since( + &self, + cursor: EventSequence, + limit: usize, + ) -> io::Result> { + let raw_events = self.reader.read_events_from(cursor, limit)?; + raw_events + .iter() + .map(|raw| { + let payload = decode_payload(&raw.payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mutation_set = payload.mutation_set.clone(); + let event = to_sequenced_event(raw, &payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(EventWithMutations { + event, + mutation_set, + }) + }) + .collect() + } + + pub fn get_event(&self, seq: EventSequence) -> io::Result> { + self.reader.read_event_at(seq)?.map_or(Ok(None), |raw| { + let payload = decode_payload(&raw.payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let event = to_sequenced_event(&raw, &payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(Some(event)) + }) + } + + pub fn max_seq(&self) -> EventSequence { + let raw = self.synced_seq.load(Ordering::Acquire); + match raw { + 0 => EventSequence::BEFORE_ALL, + n => EventSequence::new(n), + } + } + + pub fn subscribe(&self) -> broadcast::Receiver { + self.broadcast_tx.subscribe() + } + + pub fn maybe_rotate(&self) -> io::Result { + let (sealed_id, new_active_id) = { + let mut writer = self.writer.lock(); + match writer.rotate_if_needed()? { + None => return Ok(false), + Some(sealed_id) => (sealed_id, writer.active_segment_id()), + } + }; + self.reader.on_segment_rotated(sealed_id, new_active_id)?; + Ok(true) + } + + pub fn run_retention(&self, max_age: Duration) -> io::Result { + let max_age_us = u64::try_from(max_age.as_micros()).unwrap_or(u64::MAX); + let cutoff_us = TimestampMicros::now().raw().saturating_sub(max_age_us); + let active_id = self.writer.lock().active_segment_id(); + let segments = self.manager.list_segments()?; + + let deleted = segments + .iter() + .take_while(|&&id| id != active_id) + .filter(|&&id| { + self.reader + .load_index(id) + .ok() + .and_then(|idx| idx.last_seq()) + .and_then(|seq| { + self.reader + .read_event_at(seq) + .ok() + .flatten() + .map(|e| e.timestamp.raw() < cutoff_us) + }) + .unwrap_or(false) + }) + .copied() + .collect::>(); + + deleted.iter().try_for_each(|&id| -> io::Result<()> { + self.manager.delete_segment(id)?; + self.reader.invalidate_index(id); + self.reader.invalidate_mmap(id); + Ok(()) + })?; + + if !deleted.is_empty() { + self.reader.refresh_segment_ranges()?; + } + + Ok(deleted.len()) + } + + pub fn segment_count(&self) -> usize { + self.manager.list_segments().map_or(0, |s| s.len()) + } + + pub fn disk_usage(&self) -> io::Result { + let segments = self.manager.list_segments()?; + segments.iter().try_fold(0u64, |acc, &id| { + let fd = self.manager.open_for_read(id)?; + let size = self.manager.io().file_size(fd)?; + Ok(acc.saturating_add(size)) + }) + } + + pub fn shutdown(&self) -> io::Result<()> { + self.writer.lock().shutdown() + } + + pub fn subscriber(&self, start_seq: EventSequence) -> EventLogSubscriber { + EventLogSubscriber::new( + self.broadcast_tx.subscribe(), + Arc::clone(&self.reader), + start_seq, + ) + } + + pub fn reader(&self) -> &EventLogReader { + &self.reader + } + + pub fn manager(&self) -> &Arc> { + &self.manager + } + + pub fn consecutive_sync_failures(&self) -> u32 { + self.consecutive_sync_failures.load(Ordering::Relaxed) + } +} + +impl PostBlockstoreHook for EventLog { + fn on_blocks_synced(&self, _proof: &BlocksSynced) -> io::Result<()> { + match self.sync_and_broadcast() { + Ok(_) => { + self.consecutive_sync_failures.store(0, Ordering::Relaxed); + if let Err(e) = self.maybe_rotate() { + warn!(error = %e, "eventlog rotation deferred"); + } + Ok(()) + } + Err(e) => { + let count = self + .consecutive_sync_failures + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1); + warn!( + error = %e, + consecutive_failures = count, + "eventlog sync failed after blockstore commit" + ); + Err(e) + } + } + } +} + +pub struct EventLogSubscriber { + rx: broadcast::Receiver, + last_seen: EventSequence, + reader: Arc>, + backfill_buffer: VecDeque, + consecutive_lags: u32, + last_lag_time: Option, +} + +const MAX_CONSECUTIVE_LAGS_BEFORE_WARN: u32 = 3; +const LAG_WINDOW: Duration = Duration::from_secs(10); +const BACKFILL_BATCH_SIZE: usize = 1024; + +impl EventLogSubscriber { + pub fn new( + rx: broadcast::Receiver, + reader: Arc>, + start_seq: EventSequence, + ) -> Self { + Self { + rx, + last_seen: start_seq, + reader, + backfill_buffer: VecDeque::new(), + consecutive_lags: 0, + last_lag_time: None, + } + } + + pub async fn next(&mut self) -> Option { + loop { + if let Some(event) = self.backfill_buffer.pop_front() { + self.last_seen = event.seq; + self.consecutive_lags = 0; + return Some(event); + } + + match self.rx.recv().await { + Ok(event) if event.seq > self.last_seen => { + self.last_seen = event.seq; + self.consecutive_lags = 0; + return Some(event); + } + Ok(_) => continue, + Err(broadcast::error::RecvError::Lagged(n)) => { + warn!( + lagged = n, + last_seen = %self.last_seen, + "subscriber lagged, backfilling from disk" + ); + self.track_lag(); + match self.fill_backfill_buffer() { + Ok(()) => continue, + Err(e) => { + warn!(error = %e, "backfill failed"); + return None; + } + } + } + Err(broadcast::error::RecvError::Closed) => return None, + } + } + } + + fn fill_backfill_buffer(&mut self) -> io::Result<()> { + let events = self + .reader + .read_events_from(self.last_seen, BACKFILL_BATCH_SIZE)?; + events.into_iter().for_each(|event| { + self.backfill_buffer.push_back(event); + }); + Ok(()) + } + + fn track_lag(&mut self) { + let now = Instant::now(); + let in_window = self + .last_lag_time + .is_some_and(|t| now.duration_since(t) < LAG_WINDOW); + + if in_window { + self.consecutive_lags = self.consecutive_lags.saturating_add(1); + } else { + self.consecutive_lags = 1; + } + self.last_lag_time = Some(now); + + if self.consecutive_lags >= MAX_CONSECUTIVE_LAGS_BEFORE_WARN { + warn!( + consecutive_lags = self.consecutive_lags, + last_seen = %self.last_seen, + "subscriber repeatedly falling behind" + ); + } + } + + pub fn last_seen(&self) -> EventSequence { + self.last_seen + } +} + +fn valid_event_to_raw(e: &ValidEvent) -> RawEvent { + RawEvent { + seq: e.seq, + timestamp: e.timestamp, + did_hash: e.did_hash, + event_type: e.event_type, + payload: bytes::Bytes::from(e.payload.clone()), + } +} + +fn repo_event_type_to_tag(event_type: RepoEventType) -> EventTypeTag { + match event_type { + RepoEventType::Commit => EventTypeTag::COMMIT, + RepoEventType::Identity => EventTypeTag::IDENTITY, + RepoEventType::Account => EventTypeTag::ACCOUNT, + RepoEventType::Sync => EventTypeTag::SYNC, + } +} diff --git a/crates/tranquil-store/src/eventlog/notifier.rs b/crates/tranquil-store/src/eventlog/notifier.rs new file mode 100644 index 0000000..022dce0 --- /dev/null +++ b/crates/tranquil-store/src/eventlog/notifier.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tranquil_db_traits::{DbError, RepoEventNotifier, RepoEventReceiver}; + +use super::{EventLog, EventLogSubscriber, EventSequence}; +use crate::io::StorageIO; + +pub struct EventLogNotifier { + log: Arc>, +} + +impl EventLogNotifier { + pub fn new(log: Arc>) -> Self { + Self { log } + } +} + +#[async_trait] +impl RepoEventNotifier for EventLogNotifier { + async fn subscribe(&self) -> Result, DbError> { + let subscriber = self.log.subscriber(EventSequence::BEFORE_ALL); + Ok(Box::new(EventLogEventReceiver { subscriber })) + } +} + +struct EventLogEventReceiver { + subscriber: EventLogSubscriber, +} + +#[async_trait] +impl RepoEventReceiver for EventLogEventReceiver { + async fn recv(&mut self) -> Option { + self.subscriber.next().await.map(|event| event.seq.as_i64()) + } +} diff --git a/crates/tranquil-store/src/eventlog/payload.rs b/crates/tranquil-store/src/eventlog/payload.rs new file mode 100644 index 0000000..794d53a --- /dev/null +++ b/crates/tranquil-store/src/eventlog/payload.rs @@ -0,0 +1,461 @@ +use serde::{Deserialize, Serialize}; +use tranquil_db_traits::{AccountStatus, SequenceNumber, SequencedEvent}; +use tranquil_types::{CidLink, Did, Handle}; + +use crate::eventlog::reader::RawEvent; +use crate::eventlog::types::MAX_EVENT_PAYLOAD; + +const PAYLOAD_VERSION: u8 = 1; +const LARGE_PAYLOAD_WARNING_THRESHOLD: usize = 1024 * 1024; + +const CID_BYTE_LEN: usize = 36; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventPayload { + pub did: String, + pub commit_cid: Option>, + pub prev_cid: Option>, + pub prev_data_cid: Option>, + pub ops: Option>, + pub blobs: Option>, + pub blocks_cids: Option>, + pub handle: Option, + pub active: Option, + pub status: Option, + pub rev: Option, + pub mutation_set: Option>, +} + +#[derive(Debug, thiserror::Error)] +pub enum PayloadError { + #[error("payload too large: {size} bytes exceeds max {max}")] + TooLarge { size: usize, max: usize }, + #[error("deserialization failed: {0}")] + DeserializeFailed(postcard::Error), + #[error("unknown payload version: {0}")] + UnknownVersion(u8), + #[error("invalid DID in payload: {0}")] + InvalidDid(String), + #[error("invalid timestamp: {0}")] + InvalidTimestamp(u64), + #[error("invalid ops JSON in payload: {0}")] + InvalidOps(serde_json::Error), + #[error("invalid handle in payload: {0}")] + InvalidHandle(String), + #[error("invalid CID length: got {got}, expected {expected}")] + InvalidCidLength { got: usize, expected: usize }, +} + +fn cid_link_to_bytes(cid: &CidLink) -> Option> { + let c = cid.to_cid()?; + let raw = c.to_bytes(); + (raw.len() == CID_BYTE_LEN).then_some(raw) +} + +fn bytes_to_cid_link(bytes: &[u8]) -> Result, PayloadError> { + if bytes.len() != CID_BYTE_LEN { + return Err(PayloadError::InvalidCidLength { + got: bytes.len(), + expected: CID_BYTE_LEN, + }); + } + Ok(cid::Cid::read_bytes(bytes) + .ok() + .map(|c| CidLink::from_cid(&c))) +} + +fn account_status_to_u8(status: &AccountStatus) -> u8 { + match status { + AccountStatus::Active => 0, + AccountStatus::Takendown => 1, + AccountStatus::Suspended => 2, + AccountStatus::Deactivated => 3, + AccountStatus::Deleted => 4, + } +} + +fn u8_to_account_status(tag: u8) -> Option { + match tag { + 0 => Some(AccountStatus::Active), + 1 => Some(AccountStatus::Takendown), + 2 => Some(AccountStatus::Suspended), + 3 => Some(AccountStatus::Deactivated), + 4 => Some(AccountStatus::Deleted), + _ => None, + } +} + +pub fn encode_payload(event: &SequencedEvent) -> Vec { + encode_payload_with_mutations(event, None) +} + +pub fn encode_payload_with_mutations( + event: &SequencedEvent, + mutation_set: Option<&[u8]>, +) -> Vec { + let ops_bytes = event + .ops + .as_ref() + .map(|v| serde_json::to_vec(v).expect("serde_json::Value always serializes")); + + let payload = EventPayload { + did: event.did.as_str().to_owned(), + commit_cid: event.commit_cid.as_ref().and_then(cid_link_to_bytes), + prev_cid: event.prev_cid.as_ref().and_then(cid_link_to_bytes), + prev_data_cid: event.prev_data_cid.as_ref().and_then(cid_link_to_bytes), + ops: ops_bytes, + blobs: event.blobs.clone(), + blocks_cids: event.blocks_cids.clone(), + handle: event + .handle + .as_ref() + .map(|h: &Handle| h.as_str().to_owned()), + active: event.active, + status: event.status.as_ref().map(account_status_to_u8), + rev: event.rev.clone(), + mutation_set: mutation_set.map(|b| b.to_vec()), + }; + + let body = postcard::to_allocvec(&payload).expect("EventPayload serialization is infallible"); + + if body.len() > LARGE_PAYLOAD_WARNING_THRESHOLD { + tracing::warn!( + size = body.len(), + did = %event.did, + "unusually large event payload" + ); + } + + let mut buf = Vec::with_capacity(1 + body.len()); + buf.push(PAYLOAD_VERSION); + buf.extend_from_slice(&body); + buf +} + +pub fn decode_payload(bytes: &[u8]) -> Result { + let (&version, body) = bytes.split_first().ok_or(PayloadError::DeserializeFailed( + postcard::Error::DeserializeUnexpectedEnd, + ))?; + + if version != PAYLOAD_VERSION { + return Err(PayloadError::UnknownVersion(version)); + } + + postcard::from_bytes(body).map_err(PayloadError::DeserializeFailed) +} + +pub fn validate_payload_size(payload: &[u8]) -> Result<(), PayloadError> { + let max = MAX_EVENT_PAYLOAD as usize; + if payload.len() > max { + return Err(PayloadError::TooLarge { + size: payload.len(), + max, + }); + } + Ok(()) +} + +pub fn to_sequenced_event( + raw: &RawEvent, + payload: &EventPayload, +) -> Result { + let timestamp_secs = raw.timestamp.raw() / 1_000_000; + let timestamp_secs_i64 = i64::try_from(timestamp_secs) + .map_err(|_| PayloadError::InvalidTimestamp(raw.timestamp.raw()))?; + let timestamp_subsec_us = + u32::try_from(raw.timestamp.raw() % 1_000_000).expect("modulo 1M always fits u32"); + + let created_at = + chrono::DateTime::from_timestamp(timestamp_secs_i64, timestamp_subsec_us * 1_000) + .unwrap_or_default(); + + let did = Did::new(&payload.did).map_err(|_| PayloadError::InvalidDid(payload.did.clone()))?; + + let ops = payload + .ops + .as_ref() + .map(|bytes| serde_json::from_slice(bytes)) + .transpose() + .map_err(PayloadError::InvalidOps)?; + + let handle = payload + .handle + .as_ref() + .map(|h| Handle::new(h.as_str()).map_err(|_| PayloadError::InvalidHandle(h.clone()))) + .transpose()?; + + Ok(SequencedEvent { + seq: SequenceNumber::from_raw(raw.seq.as_i64()), + did, + created_at, + event_type: raw.event_type.to_repo_event_type(), + commit_cid: payload + .commit_cid + .as_deref() + .map(bytes_to_cid_link) + .transpose()? + .flatten(), + prev_cid: payload + .prev_cid + .as_deref() + .map(bytes_to_cid_link) + .transpose()? + .flatten(), + prev_data_cid: payload + .prev_data_cid + .as_deref() + .map(bytes_to_cid_link) + .transpose()? + .flatten(), + ops, + blobs: payload.blobs.clone(), + blocks_cids: payload.blocks_cids.clone(), + handle, + active: payload.active, + status: payload.status.and_then(u8_to_account_status), + rev: payload.rev.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::types::{DidHash, EventSequence, EventTypeTag, TimestampMicros}; + use bytes::Bytes; + use sha2::Digest; + use tranquil_db_traits::RepoEventType; + + fn test_did() -> Did { + Did::new("did:plc:testuser1234567890abcdef").unwrap() + } + + fn test_cid_link() -> CidLink { + let hash = sha2::Digest::finalize(sha2::Sha256::new()); + let mh = multihash::Multihash::<64>::wrap(0x12, &hash).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + #[test] + fn round_trip_minimal_payload() { + let event = SequencedEvent { + seq: SequenceNumber::from_raw(42), + did: test_did(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Account, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: Some(true), + status: Some(AccountStatus::Active), + rev: None, + }; + + let encoded = encode_payload(&event); + assert_eq!(encoded[0], PAYLOAD_VERSION); + + let decoded = decode_payload(&encoded).unwrap(); + assert_eq!(decoded.did, event.did.as_str()); + assert_eq!(decoded.active, Some(true)); + assert_eq!(decoded.status, Some(0)); + assert!(decoded.commit_cid.is_none()); + } + + #[test] + fn round_trip_full_commit_payload() { + let cid = test_cid_link(); + let ops = serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}]); + + let event = SequencedEvent { + seq: SequenceNumber::from_raw(100), + did: test_did(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: Some(cid.clone()), + prev_data_cid: Some(cid.clone()), + ops: Some(ops.clone()), + blobs: Some(vec!["bafkreibtest".to_owned()]), + blocks_cids: Some(vec!["bafyreiblock".to_owned()]), + handle: Some(Handle::new("test.bsky.social").unwrap()), + active: None, + status: None, + rev: Some("rev123".to_owned()), + }; + + let encoded = encode_payload(&event); + let decoded = decode_payload(&encoded).unwrap(); + + let raw = RawEvent { + seq: EventSequence::new(100), + timestamp: TimestampMicros::now(), + did_hash: DidHash::from_did(event.did.as_str()), + event_type: EventTypeTag::COMMIT, + payload: Bytes::from(encoded), + }; + + let reconstructed = to_sequenced_event(&raw, &decoded).unwrap(); + assert_eq!(reconstructed.did.as_str(), event.did.as_str()); + assert_eq!(reconstructed.commit_cid, event.commit_cid); + assert_eq!(reconstructed.prev_cid, event.prev_cid); + assert_eq!(reconstructed.prev_data_cid, event.prev_data_cid); + assert_eq!(reconstructed.blobs, event.blobs); + assert_eq!(reconstructed.blocks_cids, event.blocks_cids); + assert_eq!( + reconstructed.handle.as_ref().map(|h: &Handle| h.as_str()), + event.handle.as_ref().map(|h: &Handle| h.as_str()) + ); + assert_eq!(reconstructed.rev, event.rev); + assert_eq!(reconstructed.event_type, RepoEventType::Commit); + + let reconstructed_ops = reconstructed.ops.unwrap(); + assert_eq!(reconstructed_ops, ops); + } + + #[test] + fn unknown_version_rejected() { + let mut encoded = encode_payload(&SequencedEvent { + seq: SequenceNumber::from_raw(1), + did: test_did(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Identity, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: None, + }); + + encoded[0] = 99; + match decode_payload(&encoded) { + Err(PayloadError::UnknownVersion(99)) => {} + other => panic!("expected UnknownVersion(99), got {other:?}"), + } + } + + #[test] + fn empty_payload_rejected() { + match decode_payload(&[]) { + Err(PayloadError::DeserializeFailed(_)) => {} + other => panic!("expected DeserializeFailed, got {other:?}"), + } + } + + #[test] + fn validate_payload_size_accepts_within_limit() { + let data = vec![0u8; MAX_EVENT_PAYLOAD as usize]; + assert!(validate_payload_size(&data).is_ok()); + } + + #[test] + fn validate_payload_size_rejects_oversized() { + let data = vec![0u8; MAX_EVENT_PAYLOAD as usize + 1]; + match validate_payload_size(&data) { + Err(PayloadError::TooLarge { size, max }) => { + assert_eq!(size, MAX_EVENT_PAYLOAD as usize + 1); + assert_eq!(max, MAX_EVENT_PAYLOAD as usize); + } + other => panic!("expected TooLarge, got {other:?}"), + } + } + + #[test] + fn account_status_round_trip() { + let statuses = [ + AccountStatus::Active, + AccountStatus::Takendown, + AccountStatus::Suspended, + AccountStatus::Deactivated, + AccountStatus::Deleted, + ]; + + statuses.iter().for_each(|status| { + let tag = account_status_to_u8(status); + let recovered = u8_to_account_status(tag).unwrap(); + assert_eq!(&recovered, status); + }); + } + + #[test] + fn invalid_account_status_returns_none() { + assert!(u8_to_account_status(255).is_none()); + } + + #[test] + fn cid_bytes_round_trip() { + let cid = test_cid_link(); + let bytes = cid_link_to_bytes(&cid).unwrap(); + assert_eq!(bytes.len(), CID_BYTE_LEN); + let recovered = bytes_to_cid_link(&bytes).unwrap().unwrap(); + assert_eq!(cid, recovered); + } + + #[test] + fn cid_bytes_wrong_length_rejected() { + let short = vec![0u8; 10]; + match bytes_to_cid_link(&short) { + Err(PayloadError::InvalidCidLength { + got: 10, + expected: 36, + }) => {} + other => panic!("expected InvalidCidLength, got {other:?}"), + } + } + + #[test] + fn event_type_tag_mapping() { + assert_eq!( + EventTypeTag::COMMIT.to_repo_event_type(), + RepoEventType::Commit + ); + assert_eq!( + EventTypeTag::IDENTITY.to_repo_event_type(), + RepoEventType::Identity + ); + assert_eq!( + EventTypeTag::ACCOUNT.to_repo_event_type(), + RepoEventType::Account + ); + assert_eq!(EventTypeTag::SYNC.to_repo_event_type(), RepoEventType::Sync); + } + + #[test] + fn timestamp_microseconds_preserved() { + let us = 1_700_000_000_123_456u64; + let raw = RawEvent { + seq: EventSequence::new(1), + timestamp: TimestampMicros::new(us), + did_hash: DidHash::from_did("did:plc:test"), + event_type: EventTypeTag::COMMIT, + payload: Bytes::new(), + }; + + let payload = EventPayload { + did: "did:plc:testuser1234567890abcdef".to_owned(), + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: None, + mutation_set: None, + }; + + let event = to_sequenced_event(&raw, &payload).unwrap(); + let recovered_us = u64::try_from(event.created_at.timestamp_micros()).unwrap(); + assert_eq!(recovered_us, us); + } +} diff --git a/crates/tranquil-store/src/eventlog/reader.rs b/crates/tranquil-store/src/eventlog/reader.rs new file mode 100644 index 0000000..80c407f --- /dev/null +++ b/crates/tranquil-store/src/eventlog/reader.rs @@ -0,0 +1,1011 @@ +use std::cell::Cell; +use std::collections::HashMap; +use std::io; +use std::sync::Arc; + +use bytes::Bytes; +use parking_lot::RwLock; +use tracing::warn; + +use crate::io::{MappedFile, StorageIO}; + +use super::manager::SegmentManager; +use super::segment_file::{ReadEventRecord, SEGMENT_HEADER_SIZE, decode_event_record}; +use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment}; +use super::types::{ + DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros, +}; + +const FIRST_EVENT_OFFSET: SegmentOffset = SegmentOffset::new(SEGMENT_HEADER_SIZE as u64); + +#[derive(Debug, Clone)] +pub struct RawEvent { + pub seq: EventSequence, + pub timestamp: TimestampMicros, + pub did_hash: DidHash, + pub event_type: EventTypeTag, + pub payload: Bytes, +} + +#[derive(Debug, Clone, Copy)] +struct SegmentRange { + id: SegmentId, + first: EventSequence, + last: EventSequence, +} + +pub struct EventLogReader { + manager: Arc>, + indexes: RwLock>>, + ranges: RwLock>, + mmaps: RwLock>>, + active_segment: RwLock>, + use_mmap: bool, +} + +impl EventLogReader { + pub fn new(manager: Arc>, use_mmap: bool) -> Self { + Self { + manager, + indexes: RwLock::new(HashMap::new()), + ranges: RwLock::new(Vec::new()), + mmaps: RwLock::new(HashMap::new()), + active_segment: RwLock::new(None), + use_mmap, + } + } + + pub fn set_active_segment(&self, id: SegmentId) { + *self.active_segment.write() = Some(id); + } + + pub fn extend_active_range(&self, first_seq: EventSequence, last_seq: EventSequence) { + let active_id = match *self.active_segment.read() { + Some(id) => id, + None => return, + }; + + let mut ranges = self.ranges.write(); + match ranges.last_mut() { + Some(last_range) if last_range.id == active_id => { + last_range.last = last_seq; + } + _ => { + ranges.push(SegmentRange { + id: active_id, + first: first_seq, + last: last_seq, + }); + } + } + } + + pub fn seed_index(&self, segment_id: SegmentId, index: SegmentIndex) { + self.indexes.write().insert(segment_id, Arc::new(index)); + } + + pub fn load_index(&self, segment_id: SegmentId) -> io::Result> { + if let Some(idx) = self.indexes.read().get(&segment_id) { + return Ok(Arc::clone(idx)); + } + + let index = match SegmentIndex::load( + self.manager.io(), + &self.manager.index_path(segment_id), + ) { + Ok(Some(idx)) => idx, + Ok(None) => self.rebuild_index(segment_id)?, + Err(e) => { + warn!(segment = %segment_id, error = %e, "index load failed, rebuilding from segment scan"); + self.rebuild_index(segment_id)? + } + }; + + let arc = Arc::new(index); + self.indexes.write().insert(segment_id, Arc::clone(&arc)); + Ok(arc) + } + + fn rebuild_index(&self, segment_id: SegmentId) -> io::Result { + let fd = self.manager.open_for_read(segment_id)?; + let (idx, _) = rebuild_from_segment(self.manager.io(), fd, DEFAULT_INDEX_INTERVAL)?; + let _ = idx.save(self.manager.io(), &self.manager.index_path(segment_id)); + Ok(idx) + } + + pub fn find_segment_for_seq(&self, target_seq: EventSequence) -> Option { + let ranges = self.ranges.read(); + let idx = ranges.partition_point(|r| r.last < target_seq); + ranges + .get(idx) + .and_then(|r| (r.first <= target_seq).then_some(r.id)) + } + + pub fn refresh_segment_ranges(&self) -> io::Result<()> { + let segment_ids = self.manager.list_segments()?; + let active = *self.active_segment.read(); + + let new_ranges: Vec = segment_ids + .iter() + .filter_map(|&id| { + let is_active = active.is_some_and(|a| a == id); + let idx = match is_active { + true => { + if let Some(cached) = self.indexes.read().get(&id).cloned() { + Ok(cached) + } else { + self.rebuild_index(id).map(|rebuilt| { + let arc = Arc::new(rebuilt); + self.indexes.write().insert(id, Arc::clone(&arc)); + arc + }) + } + } + false => self.load_index(id), + }; + match idx { + Ok(idx) => match (idx.first_seq(), idx.last_seq()) { + (Some(first), Some(last)) => Some(SegmentRange { id, first, last }), + _ => None, + }, + Err(e) => { + warn!(segment = %id, error = %e, "failed to load index for range cache"); + None + } + } + }) + .collect(); + *self.ranges.write() = new_ranges; + Ok(()) + } + + fn is_mmap_eligible(&self, segment_id: SegmentId) -> bool { + self.use_mmap + && self + .active_segment + .read() + .is_none_or(|active| active != segment_id) + } + + fn get_mmap(&self, segment_id: SegmentId) -> io::Result> { + if let Some(m) = self.mmaps.read().get(&segment_id) { + return Ok(Arc::clone(m)); + } + + let fd = self.manager.open_for_read(segment_id)?; + let mapped = self.manager.io().mmap_file(fd)?; + let arc = Arc::new(mapped); + self.mmaps.write().insert(segment_id, Arc::clone(&arc)); + Ok(arc) + } + + fn scan_events_from_offset( + &self, + segment_id: SegmentId, + start_offset: SegmentOffset, + start_seq: EventSequence, + limit: usize, + events: &mut Vec, + predicate: impl FnMut(&EventSequence) -> bool, + ) -> io::Result { + if self.is_mmap_eligible(segment_id) { + self.scan_mmap( + segment_id, + start_offset, + start_seq, + limit, + events, + predicate, + ) + } else { + let fd = self.manager.open_for_read(segment_id)?; + let file_size = self.manager.io().file_size(fd)?; + self.scan_direct( + fd, + file_size, + start_offset, + start_seq, + limit, + events, + predicate, + ) + } + } + + fn scan_mmap( + &self, + segment_id: SegmentId, + start_offset: SegmentOffset, + start_seq: EventSequence, + limit: usize, + events: &mut Vec, + mut predicate: impl FnMut(&EventSequence) -> bool, + ) -> io::Result { + let mmap = self.get_mmap(segment_id)?; + let data: &[u8] = (*mmap).as_ref(); + let file_size = data.len() as u64; + let offset = Cell::new(start_offset); + let collected = Cell::new(0usize); + + std::iter::from_fn(|| { + let cur = offset.get(); + (cur.raw() < file_size && collected.get() < limit) + .then(|| decode_mmap_event(data, cur, file_size, segment_id)) + }) + .try_for_each(|result| -> io::Result<()> { + match result? { + MmapDecodeResult::Valid(event, next_offset) => { + offset.set(next_offset); + if event.seq > start_seq && predicate(&event.seq) { + events.push(event); + collected.set(collected.get() + 1); + } + } + MmapDecodeResult::Corrupted + | MmapDecodeResult::Truncated + | MmapDecodeResult::EndOfSegment => { + offset.set(SegmentOffset::new(file_size)); + } + } + Ok(()) + })?; + Ok(collected.get() >= limit) + } + + #[allow(clippy::too_many_arguments)] + fn scan_direct( + &self, + fd: crate::io::FileId, + file_size: u64, + start_offset: SegmentOffset, + start_seq: EventSequence, + limit: usize, + events: &mut Vec, + mut predicate: impl FnMut(&EventSequence) -> bool, + ) -> io::Result { + let offset = Cell::new(start_offset); + let collected = Cell::new(0usize); + + std::iter::from_fn(|| { + let cur = offset.get(); + (cur.raw() < file_size && collected.get() < limit) + .then(|| decode_event_record(self.manager.io(), fd, cur, file_size)) + }) + .try_for_each(|result| -> io::Result<()> { + match result? { + Some(ReadEventRecord::Valid { event, next_offset }) => { + offset.set(next_offset); + if event.seq > start_seq && predicate(&event.seq) { + events.push(RawEvent { + seq: event.seq, + timestamp: event.timestamp, + did_hash: event.did_hash, + event_type: event.event_type, + payload: Bytes::from(event.payload), + }); + collected.set(collected.get() + 1); + } + } + Some(ReadEventRecord::Corrupted { .. } | ReadEventRecord::Truncated { .. }) + | None => { + offset.set(SegmentOffset::new(file_size)); + } + } + Ok(()) + })?; + Ok(collected.get() >= limit) + } + + pub fn read_events_from( + &self, + start_seq: EventSequence, + limit: usize, + ) -> io::Result> { + if limit == 0 { + return Ok(Vec::new()); + } + + let ranges = self.ranges.read().clone(); + let start_idx = match start_seq { + EventSequence::BEFORE_ALL => Some(0), + seq => { + let point = ranges.partition_point(|r| r.last <= seq); + (point < ranges.len()).then_some(point) + } + }; + + let start_idx = match start_idx { + Some(idx) => idx, + None => return Ok(Vec::new()), + }; + + let mut events = Vec::with_capacity(limit.min(1024)); + + ranges[start_idx..].iter().enumerate().try_fold( + false, + |limit_reached, (i, range)| -> io::Result { + if limit_reached { + return Ok(true); + } + + let remaining = limit - events.len(); + let is_first = i == 0; + + let (scan_offset, effective_seq) = match (is_first, start_seq) { + (_, EventSequence::BEFORE_ALL) | (false, _) => { + (FIRST_EVENT_OFFSET, EventSequence::BEFORE_ALL) + } + (true, seq) => { + let index = self.load_index(range.id)?; + (index.lookup(seq).unwrap_or(FIRST_EVENT_OFFSET), seq) + } + }; + + self.scan_events_from_offset( + range.id, + scan_offset, + effective_seq, + remaining, + &mut events, + |_| true, + ) + }, + )?; + + Ok(events) + } + + pub fn read_event_at(&self, seq: EventSequence) -> io::Result> { + let segment_id = match self.find_segment_for_seq(seq) { + Some(id) => id, + None => return Ok(None), + }; + + let index = self.load_index(segment_id)?; + let scan_offset = index.lookup(seq).unwrap_or(FIRST_EVENT_OFFSET); + + let mut events = Vec::with_capacity(1); + self.scan_events_from_offset( + segment_id, + scan_offset, + seq.prev_or_before_all(), + 1, + &mut events, + |s| *s == seq, + )?; + + Ok(events.into_iter().next()) + } + + pub fn on_segment_rotated( + &self, + sealed_id: SegmentId, + new_active_id: SegmentId, + ) -> io::Result<()> { + self.invalidate_index(sealed_id); + self.invalidate_mmap(sealed_id); + self.set_active_segment(new_active_id); + self.refresh_segment_ranges() + } + + pub fn invalidate_mmap(&self, segment_id: SegmentId) { + self.mmaps.write().remove(&segment_id); + } + + pub fn invalidate_index(&self, segment_id: SegmentId) { + self.indexes.write().remove(&segment_id); + } +} + +enum MmapDecodeResult { + Valid(RawEvent, SegmentOffset), + Corrupted, + Truncated, + EndOfSegment, +} + +fn decode_mmap_event( + data: &[u8], + offset: SegmentOffset, + file_size: u64, + segment_id: SegmentId, +) -> io::Result { + use super::segment_file::EVENT_HEADER_SIZE; + use super::types::MAX_EVENT_PAYLOAD; + + let raw = offset.raw(); + if raw > file_size { + warn!( + segment = %segment_id, + offset = raw, + file_size, + "decode offset past file size (corrupt index?)" + ); + return Ok(MmapDecodeResult::Corrupted); + } + let remaining = file_size - raw; + if remaining == 0 { + return Ok(MmapDecodeResult::EndOfSegment); + } + + if remaining < EVENT_HEADER_SIZE as u64 { + warn!( + segment = %segment_id, + offset = raw, + remaining, + "truncated record in sealed segment: not enough bytes for header" + ); + return Ok(MmapDecodeResult::Truncated); + } + + let base = usize::try_from(raw).expect("file offset exceeds platform address space"); + let header_slice = &data[base..base + EVENT_HEADER_SIZE]; + + let seq_raw = u64::from_le_bytes(header_slice[0..8].try_into().unwrap()); + if seq_raw == 0 { + warn!( + segment = %segment_id, + offset = raw, + "corrupted record in sealed segment: seq == 0" + ); + return Ok(MmapDecodeResult::Corrupted); + } + let seq = EventSequence::new(seq_raw); + + let timestamp = + TimestampMicros::new(u64::from_le_bytes(header_slice[8..16].try_into().unwrap())); + let did_hash = DidHash::from_raw(u32::from_le_bytes(header_slice[16..20].try_into().unwrap())); + let event_type = match EventTypeTag::from_raw(header_slice[20]) { + Some(t) => t, + None => { + warn!( + segment = %segment_id, + offset = raw, + tag = header_slice[20], + "corrupted record in sealed segment: invalid event type" + ); + return Ok(MmapDecodeResult::Corrupted); + } + }; + + let payload_len = u32::from_le_bytes(header_slice[21..25].try_into().unwrap()); + if payload_len > MAX_EVENT_PAYLOAD { + warn!( + segment = %segment_id, + offset = raw, + payload_len, + "corrupted record in sealed segment: payload exceeds maximum" + ); + return Ok(MmapDecodeResult::Corrupted); + } + + let record_size = super::segment_file::EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len); + if record_size > remaining { + warn!( + segment = %segment_id, + offset = raw, + record_size, + remaining, + "truncated record in sealed segment: record extends past file end" + ); + return Ok(MmapDecodeResult::Truncated); + } + + let payload_start = base + EVENT_HEADER_SIZE; + let payload_end = payload_start + usize::try_from(payload_len).expect("payload_len fits usize"); + + let checksum_start = payload_end; + let stored_checksum = + u32::from_le_bytes(data[checksum_start..checksum_start + 4].try_into().unwrap()); + + let mut hasher = xxhash_rust::xxh3::Xxh3::new(); + hasher.update(header_slice); + hasher.update(&data[payload_start..payload_end]); + let computed = hasher.digest() as u32; + + if stored_checksum != computed { + warn!( + segment = %segment_id, + offset = raw, + seq = %seq, + stored = stored_checksum, + computed, + "corrupted record in sealed segment: checksum mismatch" + ); + return Ok(MmapDecodeResult::Corrupted); + } + + let next_offset = offset.advance(record_size); + Ok(MmapDecodeResult::Valid( + RawEvent { + seq, + timestamp, + did_hash, + event_type, + payload: Bytes::copy_from_slice(&data[payload_start..payload_end]), + }, + next_offset, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::segment_file::EVENT_RECORD_OVERHEAD; + use crate::eventlog::writer::EventLogWriter; + use crate::sim::SimulatedIO; + use std::path::PathBuf; + + fn setup_manager(max_segment_size: u64) -> Arc> { + let sim = SimulatedIO::pristine(42); + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()) + } + + fn setup_with_events( + event_count: u64, + payload_size: usize, + max_segment_size: u64, + ) -> ( + Arc>, + EventLogReader, + ) { + let mgr = setup_manager(max_segment_size); + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=event_count).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xAA; payload_size], + ) + .unwrap(); + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + (mgr, reader) + } + + fn setup_multi_segment( + events_per_segment: u64, + num_segments: u64, + payload_size: usize, + ) -> ( + Arc>, + EventLogReader, + ) { + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = + (SEGMENT_HEADER_SIZE + record_size * events_per_segment as usize) as u64; + + let mgr = setup_manager(max_segment_size); + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + let total = events_per_segment * num_segments; + (1..=total).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![i as u8; payload_size], + ) + .unwrap(); + if i % events_per_segment == 0 && i < total { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + (mgr, reader) + } + + #[test] + fn read_events_from_single_segment() { + let (_, reader) = setup_with_events(10, 50, 64 * 1024); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert_eq!(events.len(), 10); + events.iter().enumerate().for_each(|(i, e)| { + assert_eq!(e.seq, EventSequence::new(i as u64 + 1)); + assert_eq!(e.event_type, EventTypeTag::COMMIT); + assert_eq!(e.payload.len(), 50); + }); + } + + #[test] + fn read_events_from_cursor() { + let (_, reader) = setup_with_events(10, 50, 64 * 1024); + + let events = reader.read_events_from(EventSequence::new(5), 100).unwrap(); + assert_eq!(events.len(), 5); + assert_eq!(events[0].seq, EventSequence::new(6)); + assert_eq!(events[4].seq, EventSequence::new(10)); + } + + #[test] + fn read_events_respects_limit() { + let (_, reader) = setup_with_events(10, 50, 64 * 1024); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 3) + .unwrap(); + assert_eq!(events.len(), 3); + assert_eq!(events[0].seq, EventSequence::new(1)); + assert_eq!(events[2].seq, EventSequence::new(3)); + } + + #[test] + fn read_events_empty_on_zero_limit() { + let (_, reader) = setup_with_events(5, 50, 64 * 1024); + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 0) + .unwrap(); + assert!(events.is_empty()); + } + + #[test] + fn read_event_at_existing() { + let (_, reader) = setup_with_events(10, 50, 64 * 1024); + + let event = reader.read_event_at(EventSequence::new(5)).unwrap(); + assert!(event.is_some()); + let event = event.unwrap(); + assert_eq!(event.seq, EventSequence::new(5)); + assert_eq!(event.payload.len(), 50); + } + + #[test] + fn read_event_at_missing() { + let (_, reader) = setup_with_events(5, 50, 64 * 1024); + + let event = reader.read_event_at(EventSequence::new(100)).unwrap(); + assert!(event.is_none()); + } + + #[test] + fn cross_segment_read() { + let (_, reader) = setup_multi_segment(3, 3, 50); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert_eq!(events.len(), 9); + events.iter().enumerate().for_each(|(i, e)| { + assert_eq!(e.seq, EventSequence::new(i as u64 + 1)); + }); + } + + #[test] + fn cross_segment_cursor_resumption() { + let (_, reader) = setup_multi_segment(3, 3, 50); + + let events = reader.read_events_from(EventSequence::new(4), 100).unwrap(); + assert_eq!(events.len(), 5); + assert_eq!(events[0].seq, EventSequence::new(5)); + assert_eq!(events[4].seq, EventSequence::new(9)); + } + + #[test] + fn cross_segment_limit_respected() { + let (_, reader) = setup_multi_segment(3, 3, 50); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 5) + .unwrap(); + assert_eq!(events.len(), 5); + assert_eq!(events[0].seq, EventSequence::new(1)); + assert_eq!(events[4].seq, EventSequence::new(5)); + } + + #[test] + fn cross_segment_limit_at_boundary() { + let (_, reader) = setup_multi_segment(3, 3, 50); + + let events = reader.read_events_from(EventSequence::new(2), 5).unwrap(); + assert_eq!(events.len(), 5); + assert_eq!(events[0].seq, EventSequence::new(3)); + assert_eq!(events[4].seq, EventSequence::new(7)); + } + + #[test] + fn find_segment_for_seq_locates_correct_segment() { + let (_, reader) = setup_multi_segment(3, 2, 50); + + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(1)), + Some(SegmentId::new(1)) + ); + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(3)), + Some(SegmentId::new(1)) + ); + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(4)), + Some(SegmentId::new(2)) + ); + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(6)), + Some(SegmentId::new(2)) + ); + assert_eq!(reader.find_segment_for_seq(EventSequence::new(100)), None); + } + + #[test] + fn index_caching_returns_same_arc() { + let (_, reader) = setup_with_events(5, 50, 64 * 1024); + + let idx1 = reader.load_index(SegmentId::new(1)).unwrap(); + let idx2 = reader.load_index(SegmentId::new(1)).unwrap(); + assert!(Arc::ptr_eq(&idx1, &idx2)); + } + + #[test] + fn refresh_after_segment_deletion() { + let (mgr, reader) = setup_multi_segment(3, 3, 50); + + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(1)), + Some(SegmentId::new(1)) + ); + + mgr.delete_segment(SegmentId::new(1)).unwrap(); + reader.invalidate_index(SegmentId::new(1)); + reader.invalidate_mmap(SegmentId::new(1)); + reader.refresh_segment_ranges().unwrap(); + + assert_eq!(reader.find_segment_for_seq(EventSequence::new(1)), None); + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(4)), + Some(SegmentId::new(2)) + ); + } + + #[test] + fn mmap_read_matches_direct_read() { + let (mgr, direct_reader) = setup_with_events(10, 50, 64 * 1024); + + let mmap_reader = EventLogReader::new(Arc::clone(&mgr), true); + mmap_reader.refresh_segment_ranges().unwrap(); + + let direct_events = direct_reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + let mmap_events = mmap_reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + + assert_eq!(direct_events.len(), mmap_events.len()); + direct_events + .iter() + .zip(mmap_events.iter()) + .for_each(|(d, m)| { + assert_eq!(d.seq, m.seq); + assert_eq!(d.timestamp, m.timestamp); + assert_eq!(d.did_hash, m.did_hash); + assert_eq!(d.event_type, m.event_type); + assert_eq!(d.payload, m.payload); + }); + } + + #[test] + fn read_event_at_first_and_last() { + let (_, reader) = setup_with_events(20, 50, 64 * 1024); + + let first = reader + .read_event_at(EventSequence::new(1)) + .unwrap() + .unwrap(); + assert_eq!(first.seq, EventSequence::new(1)); + + let last = reader + .read_event_at(EventSequence::new(20)) + .unwrap() + .unwrap(); + assert_eq!(last.seq, EventSequence::new(20)); + } + + #[test] + fn empty_reader_returns_empty() { + let mgr = setup_manager(64 * 1024); + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert!(events.is_empty()); + + let event = reader.read_event_at(EventSequence::new(1)).unwrap(); + assert!(event.is_none()); + } + + #[test] + fn cursor_past_end_returns_empty() { + let (_, reader) = setup_with_events(5, 50, 64 * 1024); + let events = reader.read_events_from(EventSequence::new(5), 100).unwrap(); + assert!(events.is_empty()); + } + + #[test] + fn cross_segment_read_event_at() { + let (_, reader) = setup_multi_segment(3, 3, 50); + + (1..=9).for_each(|i| { + let event = reader + .read_event_at(EventSequence::new(i)) + .unwrap() + .unwrap(); + assert_eq!(event.seq, EventSequence::new(i)); + assert_eq!(event.payload[0], i as u8); + }); + } + + #[test] + fn different_event_types_preserved() { + let mgr = setup_manager(64 * 1024); + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + let types = [ + EventTypeTag::COMMIT, + EventTypeTag::IDENTITY, + EventTypeTag::ACCOUNT, + EventTypeTag::SYNC, + ]; + types.iter().enumerate().for_each(|(i, &et)| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + et, + vec![0xAA; 32], + ) + .unwrap(); + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert_eq!(events[0].event_type, EventTypeTag::COMMIT); + assert_eq!(events[1].event_type, EventTypeTag::IDENTITY); + assert_eq!(events[2].event_type, EventTypeTag::ACCOUNT); + assert_eq!(events[3].event_type, EventTypeTag::SYNC); + } + + #[test] + fn active_segment_excludes_mmap() { + let (mgr, _) = setup_with_events(10, 50, 64 * 1024); + + let reader = EventLogReader::new(Arc::clone(&mgr), true); + reader.set_active_segment(SegmentId::new(1)); + reader.refresh_segment_ranges().unwrap(); + + assert!(!reader.is_mmap_eligible(SegmentId::new(1))); + assert!(reader.is_mmap_eligible(SegmentId::new(2))); + } + + #[test] + fn no_active_segment_mmaps_all() { + let reader: EventLogReader = + EventLogReader::new(setup_manager(64 * 1024), true); + + assert!(reader.is_mmap_eligible(SegmentId::new(1))); + assert!(reader.is_mmap_eligible(SegmentId::new(99))); + } + + #[test] + fn corrupt_index_offset_does_not_panic() { + let mgr = setup_manager(64 * 1024); + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=5).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xAA; 50], + ) + .unwrap(); + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let mut bad_index = SegmentIndex::new(); + bad_index.record(EventSequence::new(1), SegmentOffset::new(999_999)); + bad_index.record(EventSequence::new(5), SegmentOffset::new(999_999)); + bad_index + .save(mgr.io(), &mgr.index_path(SegmentId::new(1))) + .unwrap(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert!(events.is_empty() || events.len() <= 5); + + let event = reader.read_event_at(EventSequence::new(3)).unwrap(); + assert!(event.is_none() || event.is_some_and(|e| e.seq == EventSequence::new(3))); + } + + #[test] + fn corrupt_index_offset_mmap_does_not_panic() { + let mgr = setup_manager(64 * 1024); + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=5).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xAA; 50], + ) + .unwrap(); + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let mut bad_index = SegmentIndex::new(); + bad_index.record(EventSequence::new(1), SegmentOffset::new(999_999)); + bad_index.record(EventSequence::new(5), SegmentOffset::new(999_999)); + bad_index + .save(mgr.io(), &mgr.index_path(SegmentId::new(1))) + .unwrap(); + + let reader = EventLogReader::new(Arc::clone(&mgr), true); + reader.refresh_segment_ranges().unwrap(); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert!(events.is_empty() || events.len() <= 5); + } + + #[test] + fn on_segment_rotated_updates_state() { + let (mgr, _direct_reader) = setup_multi_segment(3, 2, 50); + + let reader = EventLogReader::new(Arc::clone(&mgr), true); + reader.refresh_segment_ranges().unwrap(); + + assert_eq!( + reader.find_segment_for_seq(EventSequence::new(1)), + Some(SegmentId::new(1)) + ); + + reader + .on_segment_rotated(SegmentId::new(1), SegmentId::new(2)) + .unwrap(); + + assert!(!reader.is_mmap_eligible(SegmentId::new(2))); + assert!(reader.is_mmap_eligible(SegmentId::new(1))); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, 100) + .unwrap(); + assert_eq!(events.len(), 6); + } +} diff --git a/crates/tranquil-store/src/eventlog/segment_file.rs b/crates/tranquil-store/src/eventlog/segment_file.rs new file mode 100644 index 0000000..7a3ddc8 --- /dev/null +++ b/crates/tranquil-store/src/eventlog/segment_file.rs @@ -0,0 +1,921 @@ +use std::io; + +use crate::io::{FileId, StorageIO}; + +use super::types::{ + DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset, + TimestampMicros, +}; + +pub const SEGMENT_MAGIC: [u8; 4] = *b"TQEV"; +pub const SEGMENT_FORMAT_VERSION: u8 = 1; +pub const SEGMENT_HEADER_SIZE: usize = 5; + +pub const EVENT_HEADER_SIZE: usize = 8 + 8 + 4 + 1 + 4; +pub const EVENT_RECORD_OVERHEAD: usize = EVENT_HEADER_SIZE + 4; + +#[must_use] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ValidEvent { + pub seq: EventSequence, + pub timestamp: TimestampMicros, + pub did_hash: DidHash, + pub event_type: EventTypeTag, + pub payload: Vec, +} + +fn event_record_checksum(header: &[u8; EVENT_HEADER_SIZE], payload: &[u8]) -> u32 { + let mut hasher = xxhash_rust::xxh3::Xxh3::new(); + hasher.update(header); + hasher.update(payload); + hasher.digest() as u32 +} + +fn encode_header(event: &ValidEvent, payload_len: u32) -> [u8; EVENT_HEADER_SIZE] { + let mut header = [0u8; EVENT_HEADER_SIZE]; + header[0..8].copy_from_slice(&event.seq.raw().to_le_bytes()); + header[8..16].copy_from_slice(&event.timestamp.raw().to_le_bytes()); + header[16..20].copy_from_slice(&event.did_hash.raw().to_le_bytes()); + header[20] = event.event_type.raw(); + header[21..25].copy_from_slice(&payload_len.to_le_bytes()); + header +} + +pub fn encode_event_record( + io: &S, + fd: FileId, + offset: SegmentOffset, + event: &ValidEvent, +) -> io::Result { + let payload_len = u32::try_from(event.payload.len()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "event payload exceeds u32::MAX", + ) + })?; + if payload_len > MAX_EVENT_PAYLOAD { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "event payload exceeds MAX_EVENT_PAYLOAD", + )); + } + + let header = encode_header(event, payload_len); + let checksum = event_record_checksum(&header, &event.payload); + let record_size = EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len); + + let base = offset.raw(); + io.write_all_at(fd, base, &header)?; + io.write_all_at(fd, base + EVENT_HEADER_SIZE as u64, &event.payload)?; + io.write_all_at( + fd, + base + EVENT_HEADER_SIZE as u64 + u64::from(payload_len), + &checksum.to_le_bytes(), + )?; + + Ok(record_size) +} + +#[must_use] +#[derive(Debug)] +pub enum ReadEventRecord { + Valid { + event: ValidEvent, + next_offset: SegmentOffset, + }, + Corrupted { + offset: SegmentOffset, + }, + Truncated { + offset: SegmentOffset, + }, +} + +pub fn decode_event_record( + io: &S, + fd: FileId, + offset: SegmentOffset, + file_size: u64, +) -> io::Result> { + let raw = offset.raw(); + if raw > file_size { + return Ok(Some(ReadEventRecord::Corrupted { offset })); + } + let remaining = file_size - raw; + if remaining == 0 { + return Ok(None); + } + + if remaining < EVENT_HEADER_SIZE as u64 { + return Ok(Some(ReadEventRecord::Truncated { offset })); + } + + let mut header = [0u8; EVENT_HEADER_SIZE]; + io.read_exact_at(fd, raw, &mut header)?; + + let seq_raw = u64::from_le_bytes(header[0..8].try_into().unwrap()); + if seq_raw == 0 { + return Ok(Some(ReadEventRecord::Corrupted { offset })); + } + let seq = EventSequence::new(seq_raw); + + let timestamp = TimestampMicros::new(u64::from_le_bytes(header[8..16].try_into().unwrap())); + let did_hash = DidHash::from_raw(u32::from_le_bytes(header[16..20].try_into().unwrap())); + let event_type_raw = header[20]; + let event_type = match EventTypeTag::from_raw(event_type_raw) { + Some(t) => t, + None => return Ok(Some(ReadEventRecord::Corrupted { offset })), + }; + + let payload_len = u32::from_le_bytes(header[21..25].try_into().unwrap()); + if payload_len > MAX_EVENT_PAYLOAD { + return Ok(Some(ReadEventRecord::Corrupted { offset })); + } + + let record_size = EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len); + if record_size > remaining { + return Ok(Some(ReadEventRecord::Truncated { offset })); + } + + let payload_offset = raw + EVENT_HEADER_SIZE as u64; + let mut payload = vec![0u8; usize::try_from(payload_len).expect("payload_len fits usize")]; + io.read_exact_at(fd, payload_offset, &mut payload)?; + + let mut checksum_bytes = [0u8; 4]; + io.read_exact_at( + fd, + payload_offset + u64::from(payload_len), + &mut checksum_bytes, + )?; + + let stored_checksum = u32::from_le_bytes(checksum_bytes); + let computed_checksum = event_record_checksum(&header, &payload); + + if stored_checksum != computed_checksum { + return Ok(Some(ReadEventRecord::Corrupted { offset })); + } + + let next_offset = offset.advance(record_size); + + Ok(Some(ReadEventRecord::Valid { + event: ValidEvent { + seq, + timestamp, + did_hash, + event_type, + payload, + }, + next_offset, + })) +} + +#[derive(Debug)] +pub enum ValidateEventRecord { + Valid { + seq: EventSequence, + next_offset: SegmentOffset, + }, + Corrupted, + Truncated, +} + +const CHECKSUM_CHUNK_SIZE: usize = 8 * 1024; + +pub fn validate_event_record( + io: &S, + fd: FileId, + offset: SegmentOffset, + file_size: u64, +) -> io::Result> { + let raw = offset.raw(); + assert!( + raw <= file_size, + "validate offset {raw} past file size {file_size}" + ); + let remaining = file_size - raw; + if remaining == 0 { + return Ok(None); + } + + if remaining < EVENT_HEADER_SIZE as u64 { + return Ok(Some(ValidateEventRecord::Truncated)); + } + + let mut header = [0u8; EVENT_HEADER_SIZE]; + io.read_exact_at(fd, raw, &mut header)?; + + let seq_raw = u64::from_le_bytes(header[0..8].try_into().unwrap()); + if seq_raw == 0 { + return Ok(Some(ValidateEventRecord::Corrupted)); + } + let seq = EventSequence::new(seq_raw); + + let event_type_raw = header[20]; + if EventTypeTag::from_raw(event_type_raw).is_none() { + return Ok(Some(ValidateEventRecord::Corrupted)); + } + + let payload_len = u32::from_le_bytes(header[21..25].try_into().unwrap()); + if payload_len > MAX_EVENT_PAYLOAD { + return Ok(Some(ValidateEventRecord::Corrupted)); + } + + let record_size = EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len); + if record_size > remaining { + return Ok(Some(ValidateEventRecord::Truncated)); + } + + let payload_offset = raw + EVENT_HEADER_SIZE as u64; + + let mut hasher = xxhash_rust::xxh3::Xxh3::new(); + hasher.update(&header); + let mut chunk = [0u8; CHECKSUM_CHUNK_SIZE]; + (0..u64::from(payload_len)) + .step_by(CHECKSUM_CHUNK_SIZE) + .map(|chunk_start| { + let to_read = + ((u64::from(payload_len) - chunk_start) as usize).min(CHECKSUM_CHUNK_SIZE); + (payload_offset + chunk_start, to_read) + }) + .try_for_each(|(pos, to_read)| { + io.read_exact_at(fd, pos, &mut chunk[..to_read])?; + hasher.update(&chunk[..to_read]); + Ok::<_, io::Error>(()) + })?; + let computed_checksum = hasher.digest() as u32; + + let mut checksum_bytes = [0u8; 4]; + io.read_exact_at( + fd, + payload_offset + u64::from(payload_len), + &mut checksum_bytes, + )?; + let stored_checksum = u32::from_le_bytes(checksum_bytes); + + if stored_checksum != computed_checksum { + return Ok(Some(ValidateEventRecord::Corrupted)); + } + + let next_offset = offset.advance(record_size); + Ok(Some(ValidateEventRecord::Valid { seq, next_offset })) +} + +pub struct SegmentWriter { + fd: FileId, + segment_id: SegmentId, + position: SegmentOffset, + base_seq: EventSequence, + last_seq: Option, +} + +impl SegmentWriter { + pub fn new( + io: &S, + fd: FileId, + segment_id: SegmentId, + base_seq: EventSequence, + ) -> io::Result { + let mut header = [0u8; SEGMENT_HEADER_SIZE]; + header[..4].copy_from_slice(&SEGMENT_MAGIC); + header[4] = SEGMENT_FORMAT_VERSION; + io.write_all_at(fd, 0, &header)?; + Ok(Self { + fd, + segment_id, + position: SegmentOffset::new(SEGMENT_HEADER_SIZE as u64), + base_seq, + last_seq: None, + }) + } + + pub fn resume( + io: &S, + fd: FileId, + segment_id: SegmentId, + position: SegmentOffset, + base_seq: EventSequence, + last_seq: Option, + ) -> Self { + assert!( + position.raw() >= SEGMENT_HEADER_SIZE as u64, + "resume position {position:?} is before header end" + ); + #[cfg(debug_assertions)] + { + let mut magic = [0u8; 4]; + io.read_exact_at(fd, 0, &mut magic) + .expect("resume: failed to read segment header"); + assert_eq!(magic, SEGMENT_MAGIC, "resume: bad segment magic"); + } + #[cfg(not(debug_assertions))] + let _ = io; + Self { + fd, + segment_id, + position, + base_seq, + last_seq, + } + } + + pub fn append_event( + &mut self, + io: &S, + event: &ValidEvent, + ) -> io::Result { + assert!( + self.last_seq.is_none_or(|prev| event.seq > prev), + "non-monotonic sequence: {} after {}", + event.seq, + self.last_seq.unwrap() + ); + let record_offset = self.position; + let bytes_written = encode_event_record(io, self.fd, record_offset, event)?; + self.position = self.position.advance(bytes_written); + self.last_seq = Some(event.seq); + Ok(record_offset) + } + + pub fn sync(&self, io: &S) -> io::Result<()> { + io.sync(self.fd) + } + + pub fn position(&self) -> SegmentOffset { + self.position + } + + pub fn segment_id(&self) -> SegmentId { + self.segment_id + } + + pub fn base_seq(&self) -> EventSequence { + self.base_seq + } + + pub fn fd(&self) -> FileId { + self.fd + } +} + +pub struct SegmentReader<'a, S: StorageIO> { + io: &'a S, + fd: FileId, + position: SegmentOffset, + file_size: u64, +} + +impl<'a, S: StorageIO> SegmentReader<'a, S> { + pub fn open(io: &'a S, fd: FileId) -> io::Result { + let file_size = io.file_size(fd)?; + if file_size < SEGMENT_HEADER_SIZE as u64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file too small for segment header", + )); + } + + let mut header = [0u8; SEGMENT_HEADER_SIZE]; + io.read_exact_at(fd, 0, &mut header)?; + + if header[..SEGMENT_MAGIC.len()] != SEGMENT_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "bad segment magic", + )); + } + if header[SEGMENT_MAGIC.len()] != SEGMENT_FORMAT_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported segment format version", + )); + } + + Ok(Self { + io, + fd, + position: SegmentOffset::new(SEGMENT_HEADER_SIZE as u64), + file_size, + }) + } + + pub fn valid_prefix(self) -> io::Result> { + self.map(|result| { + result.map(|record| match record { + ReadEventRecord::Valid { event, .. } => Some(event), + ReadEventRecord::Corrupted { .. } | ReadEventRecord::Truncated { .. } => None, + }) + }) + .scan((), |(), result| match result { + Err(e) => Some(Err(e)), + Ok(Some(event)) => Some(Ok(event)), + Ok(None) => None, + }) + .collect() + } + + pub fn fd(&self) -> FileId { + self.fd + } + + pub fn position(&self) -> SegmentOffset { + self.position + } + + pub fn file_size(&self) -> u64 { + self.file_size + } +} + +impl Iterator for SegmentReader<'_, S> { + type Item = io::Result; + + fn next(&mut self) -> Option { + match decode_event_record(self.io, self.fd, self.position, self.file_size) { + Err(e) => { + self.position = SegmentOffset::new(self.file_size); + Some(Err(e)) + } + Ok(None) => None, + Ok(Some(record)) => { + match &record { + ReadEventRecord::Valid { next_offset, .. } => { + self.position = *next_offset; + } + ReadEventRecord::Corrupted { .. } | ReadEventRecord::Truncated { .. } => { + self.position = SegmentOffset::new(self.file_size); + } + } + Some(Ok(record)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::OpenOptions; + use crate::sim::SimulatedIO; + use proptest::prelude::*; + use std::path::Path; + + fn setup() -> (SimulatedIO, FileId) { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + let fd = sim + .open(Path::new("/test/segment.tqe"), OpenOptions::read_write()) + .unwrap(); + (sim, fd) + } + + fn test_did_hash(seed: u8) -> DidHash { + DidHash::from_did(&format!("did:plc:test{seed}")) + } + + fn test_event(seq: u64, payload: &[u8]) -> ValidEvent { + ValidEvent { + seq: EventSequence::new(seq), + timestamp: TimestampMicros::new(seq * 1_000_000), + did_hash: test_did_hash(seq as u8), + event_type: EventTypeTag::COMMIT, + payload: payload.to_vec(), + } + } + + #[test] + fn write_and_read_single_event() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let event = test_event(1, b"test event payload"); + let offset = writer.append_event(&sim, &event).unwrap(); + writer.sync(&sim).unwrap(); + + assert_eq!(offset, SegmentOffset::new(SEGMENT_HEADER_SIZE as u64)); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0], event); + } + + #[test] + fn write_and_read_multiple_events() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let written: Vec = (1u64..=3) + .map(|i| { + let event = test_event(i, format!("event {i}").as_bytes()); + writer.append_event(&sim, &event).unwrap(); + event + }) + .collect(); + writer.sync(&sim).unwrap(); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events, written); + } + + #[test] + fn empty_segment_has_no_events() { + let (sim, fd) = setup(); + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert!(events.is_empty()); + } + + #[test] + fn detects_truncated_event() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer + .append_event(&sim, &test_event(1, b"complete event")) + .unwrap(); + writer.sync(&sim).unwrap(); + + sim.write_all_at(fd, writer.position().raw(), &[1, 2, 3, 4, 5]) + .unwrap(); + sim.sync(fd).unwrap(); + + let mut reader = SegmentReader::open(&sim, fd).unwrap(); + let first = reader.next().unwrap().unwrap(); + assert!(matches!(first, ReadEventRecord::Valid { .. })); + + let second = reader.next().unwrap().unwrap(); + assert!(matches!(second, ReadEventRecord::Truncated { .. })); + } + + #[test] + fn checksum_detects_corruption() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer + .append_event(&sim, &test_event(1, &vec![0xAA; 256])) + .unwrap(); + writer.sync(&sim).unwrap(); + + let corrupt_offset = SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64 + 128; + sim.write_all_at(fd, corrupt_offset, &[0x00]).unwrap(); + + let mut reader = SegmentReader::open(&sim, fd).unwrap(); + let record = reader.next().unwrap().unwrap(); + assert!(matches!(record, ReadEventRecord::Corrupted { .. })); + } + + #[test] + fn crash_before_sync_loses_events() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer + .append_event(&sim, &test_event(1, b"synced")) + .unwrap(); + writer.sync(&sim).unwrap(); + sim.sync_dir(Path::new("/test")).unwrap(); + + writer + .append_event(&sim, &test_event(2, b"not synced")) + .unwrap(); + + sim.crash(); + + let fd = sim + .open(Path::new("/test/segment.tqe"), OpenOptions::read()) + .unwrap(); + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].payload, b"synced"); + } + + #[test] + fn rejects_oversized_payload() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + let result = writer.append_event( + &sim, + &test_event(1, &vec![0u8; MAX_EVENT_PAYLOAD as usize + 1]), + ); + assert!(result.is_err()); + } + + #[test] + fn zero_length_payload_round_trips() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + let event = ValidEvent { + seq: EventSequence::new(1), + timestamp: TimestampMicros::new(1_000_000), + did_hash: test_did_hash(1), + event_type: EventTypeTag::IDENTITY, + payload: vec![], + }; + writer.append_event(&sim, &event).unwrap(); + writer.sync(&sim).unwrap(); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events, vec![event]); + } + + #[test] + fn accepts_exact_max_payload() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + let result = writer.append_event( + &sim, + &test_event(1, &vec![0xBB; MAX_EVENT_PAYLOAD as usize]), + ); + assert!(result.is_ok()); + } + + #[test] + fn bad_magic_rejected() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + let fd = sim + .open(Path::new("/test/bad.tqe"), OpenOptions::read_write()) + .unwrap(); + sim.write_all_at(fd, 0, b"NOPE\x01").unwrap(); + + let result = SegmentReader::open(&sim, fd); + assert!(result.is_err()); + } + + #[test] + fn encode_decode_round_trip_at_offset() { + let (sim, fd) = setup(); + + sim.write_all_at(fd, 0, &[0u8; 100]).unwrap(); + + let offset = SegmentOffset::new(100); + let event = ValidEvent { + seq: EventSequence::new(42), + timestamp: TimestampMicros::new(9_999_999), + did_hash: test_did_hash(7), + event_type: EventTypeTag::ACCOUNT, + payload: b"round trip test data".to_vec(), + }; + let bytes_written = encode_event_record(&sim, fd, offset, &event).unwrap(); + let expected_size = EVENT_RECORD_OVERHEAD as u64 + event.payload.len() as u64; + assert_eq!(bytes_written, expected_size); + + let file_size = sim.file_size(fd).unwrap(); + let record = decode_event_record(&sim, fd, offset, file_size) + .unwrap() + .unwrap(); + match record { + ReadEventRecord::Valid { event: decoded, .. } => assert_eq!(decoded, event), + other => panic!("expected Valid, got {other:?}"), + } + } + + #[test] + fn resume_writer_continues_at_position() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + writer.append_event(&sim, &test_event(1, b"first")).unwrap(); + writer.sync(&sim).unwrap(); + + let resume_pos = writer.position(); + let mut writer2 = SegmentWriter::resume( + &sim, + fd, + SegmentId::new(1), + resume_pos, + EventSequence::new(1), + Some(EventSequence::new(1)), + ); + writer2 + .append_event(&sim, &test_event(2, b"second")) + .unwrap(); + writer2.sync(&sim).unwrap(); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].payload, b"first"); + assert_eq!(events[1].payload, b"second"); + } + + #[test] + fn all_event_types_round_trip() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let types = [ + EventTypeTag::COMMIT, + EventTypeTag::IDENTITY, + EventTypeTag::ACCOUNT, + EventTypeTag::SYNC, + ]; + + types.iter().enumerate().for_each(|(i, &event_type)| { + let event = ValidEvent { + seq: EventSequence::new((i + 1) as u64), + timestamp: TimestampMicros::new(1_000_000), + did_hash: test_did_hash(i as u8), + event_type, + payload: b"payload".to_vec(), + }; + writer.append_event(&sim, &event).unwrap(); + }); + writer.sync(&sim).unwrap(); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events.len(), 4); + events + .iter() + .zip(types.iter()) + .for_each(|(event, &expected_type)| { + assert_eq!(event.event_type, expected_type); + }); + } + + #[test] + fn seq_zero_detected_as_corrupted() { + let (sim, fd) = setup(); + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let mut raw_header = [0u8; EVENT_HEADER_SIZE]; + raw_header[0..8].copy_from_slice(&0u64.to_le_bytes()); + raw_header[8..16].copy_from_slice(&1_000_000u64.to_le_bytes()); + raw_header[16..20].copy_from_slice(&test_did_hash(1).raw().to_le_bytes()); + raw_header[20] = EventTypeTag::COMMIT.raw(); + raw_header[21..25].copy_from_slice(&5u32.to_le_bytes()); + + sim.write_all_at(fd, SEGMENT_HEADER_SIZE as u64, &raw_header) + .unwrap(); + sim.write_all_at( + fd, + SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64, + b"hello", + ) + .unwrap(); + sim.write_all_at( + fd, + SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64 + 5, + &[0u8; 4], + ) + .unwrap(); + + let mut reader = SegmentReader::open(&sim, fd).unwrap(); + let record = reader.next().unwrap().unwrap(); + assert!(matches!(record, ReadEventRecord::Corrupted { .. })); + } + + #[test] + fn writer_accessors() { + let (sim, fd) = setup(); + let writer = + SegmentWriter::new(&sim, fd, SegmentId::new(7), EventSequence::new(100)).unwrap(); + assert_eq!(writer.segment_id(), SegmentId::new(7)); + assert_eq!(writer.base_seq(), EventSequence::new(100)); + assert_eq!( + writer.position(), + SegmentOffset::new(SEGMENT_HEADER_SIZE as u64) + ); + assert_eq!(writer.fd(), fd); + } + + fn run_crash_recovery_seed(seed: u64) { + let sim = SimulatedIO::new(seed, crate::FaultConfig::aggressive()); + let dir = Path::new("/data"); + let _ = sim.mkdir(dir); + let _ = sim.sync_dir(dir); + + let written_count = + if let Ok(fd) = sim.open(Path::new("/data/segment.tqe"), OpenOptions::read_write()) { + if let Ok(mut writer) = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)) + { + let count = (1u64..=20).fold(0u64, |count, i| { + let event = ValidEvent { + seq: EventSequence::new(i), + timestamp: TimestampMicros::new(i * 1_000_000), + did_hash: DidHash::from_did(&format!("did:plc:user{i}")), + event_type: EventTypeTag::COMMIT, + payload: vec![i as u8; ((i as usize) + 1) * 10], + }; + match writer.append_event(&sim, &event) { + Ok(_) => count + 1, + Err(_) => count, + } + }); + let _ = writer.sync(&sim); + count + } else { + 0 + } + } else { + 0 + }; + let _ = sim.sync_dir(dir); + + sim.crash(); + + if let Ok(fd) = sim.open(Path::new("/data/segment.tqe"), OpenOptions::read()) + && let Ok(reader) = SegmentReader::open(&sim, fd) + { + let recovered: Vec<_> = reader + .map_while(|r| match r { + Ok(ReadEventRecord::Valid { event, .. }) => Some(event), + _ => None, + }) + .collect(); + + assert!( + recovered.len() as u64 <= written_count, + "recovered {} events but only wrote {written_count}", + recovered.len() + ); + + recovered.windows(2).enumerate().for_each(|(i, pair)| { + assert!( + pair[0].seq < pair[1].seq, + "event {i} seq {} not less than event {} seq {}", + pair[0].seq, + i + 1, + pair[1].seq, + ); + }); + } + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + #[test] + fn sim_crash_recovery_aggressive_faults(seed in 0u64..u64::MAX) { + run_crash_recovery_seed(seed); + } + } + + fn run_bit_flip_detection_seed(seed: u64) { + let sim = SimulatedIO::pristine(seed); + let dir = Path::new("/data"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let fd = sim + .open(Path::new("/data/segment.tqe"), OpenOptions::read_write()) + .unwrap(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let data_len = ((seed % 256) as usize).max(1); + let event = ValidEvent { + seq: EventSequence::new(1), + timestamp: TimestampMicros::new(1_000_000), + did_hash: DidHash::from_did("did:plc:bitflip"), + event_type: EventTypeTag::COMMIT, + payload: vec![0xAA; data_len], + }; + writer.append_event(&sim, &event).unwrap(); + writer.sync(&sim).unwrap(); + + let record_start = SEGMENT_HEADER_SIZE as u64; + let record_end = record_start + EVENT_RECORD_OVERHEAD as u64 + data_len as u64; + let flip_pos = record_start + (seed.wrapping_mul(7) % (record_end - record_start)); + let flip_bit = (seed.wrapping_mul(13) % 8) as u8; + + let mut byte_buf = [0u8; 1]; + sim.read_exact_at(fd, flip_pos, &mut byte_buf).unwrap(); + byte_buf[0] ^= 1 << flip_bit; + sim.write_all_at(fd, flip_pos, &byte_buf).unwrap(); + + let mut reader = SegmentReader::open(&sim, fd).unwrap(); + let record = reader.next().unwrap().unwrap(); + assert!( + !matches!(record, ReadEventRecord::Valid { .. }), + "bit flip at offset {flip_pos} bit {flip_bit} was not detected" + ); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + #[test] + fn sim_bit_flip_detected_by_checksum(seed in 0u64..u64::MAX) { + run_bit_flip_detection_seed(seed); + } + } +} diff --git a/crates/tranquil-store/src/eventlog/segment_index.rs b/crates/tranquil-store/src/eventlog/segment_index.rs new file mode 100644 index 0000000..9bb77cb --- /dev/null +++ b/crates/tranquil-store/src/eventlog/segment_index.rs @@ -0,0 +1,666 @@ +use std::cell::Cell; +use std::io; +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::io::{FileId, OpenOptions, StorageIO}; +use crate::record::{RecordReader, RecordWriter}; + +use super::segment_file::{ + SEGMENT_HEADER_SIZE, SEGMENT_MAGIC, ValidateEventRecord, validate_event_record, +}; +use super::types::{EventSequence, SegmentOffset}; + +pub const DEFAULT_INDEX_INTERVAL: usize = 256; +const MAX_INDEX_ENTRIES: usize = 4 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +struct IndexEntry { + seq: EventSequence, + offset: SegmentOffset, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SegmentIndex { + entries: Vec, +} + +impl SegmentIndex { + pub fn new() -> Self { + Self { + entries: Vec::new(), + } + } + + pub fn record(&mut self, seq: EventSequence, offset: SegmentOffset) { + debug_assert!( + self.entries.last().is_none_or(|last| seq > last.seq), + "index entries must be monotonically increasing" + ); + self.entries.push(IndexEntry { seq, offset }); + } + + pub fn lookup(&self, target_seq: EventSequence) -> Option { + let idx = self.entries.partition_point(|e| e.seq <= target_seq); + match idx { + 0 => None, + i => Some(self.entries[i - 1].offset), + } + } + + pub fn first_seq(&self) -> Option { + self.entries.first().map(|e| e.seq) + } + + pub fn last_seq(&self) -> Option { + self.entries.last().map(|e| e.seq) + } + + pub fn entry_count(&self) -> usize { + self.entries.len() + } + + pub fn save(&self, io: &S, path: &Path) -> io::Result<()> { + let tmp_path = path.with_extension("tqi.tmp"); + let fd = io.open(&tmp_path, OpenOptions::read_write())?; + + let result = (|| { + let serialized = postcard::to_allocvec(&self.entries) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + let mut writer = RecordWriter::new(io, fd)?; + writer.append(&serialized)?; + io.truncate(fd, writer.position())?; + writer.sync()?; + Ok(()) + })(); + + if let Err(e) = result { + let _ = io.close(fd); + return Err(e); + } + io.close(fd)?; + + io.rename(&tmp_path, path)?; + + if let Some(parent) = path.parent() { + io.sync_dir(parent)?; + } + + Ok(()) + } + + pub fn load(io: &S, path: &Path) -> io::Result> { + let fd = match io.open(path, OpenOptions::read_only_existing()) { + Ok(fd) => fd, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + + let reader = match RecordReader::open(io, fd) { + Ok(r) => r, + Err(e) => { + let _ = io.close(fd); + return Err(e); + } + }; + let records = reader.valid_records(); + io.close(fd)?; + + let payload = records.into_iter().next().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "index file contains no records") + })?; + + const MIN_POSTCARD_ENTRY_BYTES: usize = 2; + if payload.len() / MIN_POSTCARD_ENTRY_BYTES > MAX_INDEX_ENTRIES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "index payload too large", + )); + } + + let entries: Vec = postcard::from_bytes(&payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + if entries.len() > MAX_INDEX_ENTRIES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "index contains too many entries", + )); + } + + let is_sorted = entries.windows(2).all(|pair| pair[0].seq < pair[1].seq); + if !is_sorted { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "index entries not monotonically sorted", + )); + } + + Ok(Some(Self { entries })) + } +} + +impl Default for SegmentIndex { + fn default() -> Self { + Self::new() + } +} + +struct ScanState { + index: SegmentIndex, + event_count: usize, + last_seq: Option, + last_offset: Option, +} + +pub fn rebuild_from_segment( + io: &S, + segment_fd: FileId, + index_interval: usize, +) -> io::Result<(SegmentIndex, Option)> { + assert!(index_interval > 0, "index_interval must be positive"); + let file_size = io.file_size(segment_fd)?; + + if file_size < SEGMENT_HEADER_SIZE as u64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file too small for segment header", + )); + } + + let mut header = [0u8; SEGMENT_HEADER_SIZE]; + io.read_exact_at(segment_fd, 0, &mut header)?; + if header[..SEGMENT_MAGIC.len()] != SEGMENT_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "bad segment magic", + )); + } + if header[SEGMENT_MAGIC.len()] != super::segment_file::SEGMENT_FORMAT_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported segment format version", + )); + } + + let current_offset = Cell::new(SegmentOffset::new(SEGMENT_HEADER_SIZE as u64)); + let prev_seq: Cell> = Cell::new(None); + + let mut valid_events = std::iter::from_fn(|| { + let offset = current_offset.get(); + if offset.raw() >= file_size { + return None; + } + match validate_event_record(io, segment_fd, offset, file_size) { + Err(e) => Some(Err(e)), + Ok(None) => None, + Ok(Some(ValidateEventRecord::Valid { seq, next_offset })) => { + if prev_seq.get().is_some_and(|prev| seq <= prev) { + return None; + } + prev_seq.set(Some(seq)); + current_offset.set(next_offset); + Some(Ok((seq, offset))) + } + Ok(Some(ValidateEventRecord::Corrupted | ValidateEventRecord::Truncated)) => None, + } + }); + + let initial = ScanState { + index: SegmentIndex::new(), + event_count: 0, + last_seq: None, + last_offset: None, + }; + + let state = valid_events.try_fold(initial, |mut state, record| -> io::Result { + let (seq, record_offset) = record?; + + let should_index = state.event_count == 0 || state.event_count % index_interval == 0; + if should_index { + state.index.record(seq, record_offset); + } + + state.event_count += 1; + state.last_seq = Some(seq); + state.last_offset = Some(record_offset); + + Ok(state) + })?; + + let mut index = state.index; + if let (Some(seq), Some(offset)) = (state.last_seq, state.last_offset) + && index.last_seq() != Some(seq) + { + index.record(seq, offset); + } + + let valid_end = current_offset.get().raw(); + if valid_end < file_size { + io.truncate(segment_fd, valid_end)?; + io.sync(segment_fd)?; + } + + Ok((index, state.last_seq)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::OpenOptions; + use crate::eventlog::segment_file::{ + EVENT_HEADER_SIZE, SegmentWriter, ValidEvent, encode_event_record, + }; + use crate::eventlog::types::{ + DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros, + }; + use crate::sim::SimulatedIO; + use std::path::Path; + + fn setup() -> (SimulatedIO, FileId) { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + let fd = sim + .open(Path::new("/test/segment.tqe"), OpenOptions::read_write()) + .unwrap(); + (sim, fd) + } + + fn test_event(seq: u64, payload: &[u8]) -> ValidEvent { + ValidEvent { + seq: EventSequence::new(seq), + timestamp: TimestampMicros::new(seq * 1_000_000), + did_hash: DidHash::from_did(&format!("did:plc:test{seq}")), + event_type: EventTypeTag::COMMIT, + payload: payload.to_vec(), + } + } + + fn write_n_events( + io: &S, + fd: FileId, + count: u64, + ) -> Vec<(EventSequence, SegmentOffset)> { + let mut writer = + SegmentWriter::new(io, fd, SegmentId::new(0), EventSequence::new(1)).unwrap(); + let offsets: Vec<_> = (1..=count) + .map(|i| { + let event = test_event(i, format!("payload-{i}").as_bytes()); + let offset = writer.append_event(io, &event).unwrap(); + (event.seq, offset) + }) + .collect(); + writer.sync(io).unwrap(); + offsets + } + + #[test] + fn empty_index() { + let index = SegmentIndex::new(); + assert_eq!(index.entry_count(), 0); + assert_eq!(index.first_seq(), None); + assert_eq!(index.last_seq(), None); + assert_eq!(index.lookup(EventSequence::new(1)), None); + } + + #[test] + fn record_and_lookup_single_entry() { + let mut index = SegmentIndex::new(); + index.record(EventSequence::new(10), SegmentOffset::new(100)); + + assert_eq!(index.entry_count(), 1); + assert_eq!(index.first_seq(), Some(EventSequence::new(10))); + assert_eq!(index.last_seq(), Some(EventSequence::new(10))); + + assert_eq!( + index.lookup(EventSequence::new(10)), + Some(SegmentOffset::new(100)) + ); + assert_eq!( + index.lookup(EventSequence::new(15)), + Some(SegmentOffset::new(100)) + ); + assert_eq!(index.lookup(EventSequence::new(5)), None); + } + + #[test] + fn lookup_returns_floor_entry() { + let mut index = SegmentIndex::new(); + index.record(EventSequence::new(1), SegmentOffset::new(100)); + index.record(EventSequence::new(100), SegmentOffset::new(5000)); + index.record(EventSequence::new(200), SegmentOffset::new(10000)); + + assert_eq!( + index.lookup(EventSequence::new(1)), + Some(SegmentOffset::new(100)) + ); + assert_eq!( + index.lookup(EventSequence::new(50)), + Some(SegmentOffset::new(100)) + ); + assert_eq!( + index.lookup(EventSequence::new(100)), + Some(SegmentOffset::new(5000)) + ); + assert_eq!( + index.lookup(EventSequence::new(150)), + Some(SegmentOffset::new(5000)) + ); + assert_eq!( + index.lookup(EventSequence::new(200)), + Some(SegmentOffset::new(10000)) + ); + assert_eq!( + index.lookup(EventSequence::new(999)), + Some(SegmentOffset::new(10000)) + ); + } + + #[test] + fn lookup_before_first_returns_none() { + let mut index = SegmentIndex::new(); + index.record(EventSequence::new(10), SegmentOffset::new(100)); + index.record(EventSequence::new(20), SegmentOffset::new(200)); + + assert_eq!(index.lookup(EventSequence::new(5)), None); + assert_eq!(index.lookup(EventSequence::new(9)), None); + } + + #[test] + fn save_and_load_round_trip() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let mut index = SegmentIndex::new(); + index.record(EventSequence::new(1), SegmentOffset::new(5)); + index.record(EventSequence::new(256), SegmentOffset::new(50000)); + index.record(EventSequence::new(512), SegmentOffset::new(100000)); + + let path = Path::new("/test/00000001.tqi"); + index.save(&sim, path).unwrap(); + + let loaded = SegmentIndex::load(&sim, path).unwrap().unwrap(); + assert_eq!(loaded, index); + } + + #[test] + fn load_missing_file_returns_none() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let result = SegmentIndex::load(&sim, Path::new("/test/missing.tqi")).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn load_corrupt_file_returns_err() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let path = Path::new("/test/corrupt.tqi"); + let fd = sim.open(path, OpenOptions::read_write()).unwrap(); + sim.write_all_at(fd, 0, b"TQST\x02garbage_not_valid_postcard") + .unwrap(); + sim.sync(fd).unwrap(); + sim.close(fd).unwrap(); + + let result = SegmentIndex::load(&sim, path); + assert!(result.is_err()); + } + + #[test] + fn save_empty_index_round_trips() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let index = SegmentIndex::new(); + let path = Path::new("/test/empty.tqi"); + index.save(&sim, path).unwrap(); + + let loaded = SegmentIndex::load(&sim, path).unwrap().unwrap(); + assert_eq!(loaded.entry_count(), 0); + } + + #[test] + fn rebuild_empty_segment() { + let (sim, fd) = setup(); + SegmentWriter::new(&sim, fd, SegmentId::new(0), EventSequence::new(1)).unwrap(); + sim.sync(fd).unwrap(); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(index.entry_count(), 0); + assert_eq!(last_seq, None); + } + + #[test] + fn rebuild_single_event() { + let (sim, fd) = setup(); + let offsets = write_n_events(&sim, fd, 1); + sim.sync(fd).unwrap(); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(1))); + assert_eq!(index.entry_count(), 1); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + assert_eq!(index.lookup(EventSequence::new(1)), Some(offsets[0].1)); + } + + #[test] + fn rebuild_indexes_first_and_last() { + let (sim, fd) = setup(); + let offsets = write_n_events(&sim, fd, 10); + sim.sync(fd).unwrap(); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(10))); + assert_eq!(index.entry_count(), 2); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + assert_eq!(index.last_seq(), Some(EventSequence::new(10))); + assert_eq!(index.lookup(EventSequence::new(1)), Some(offsets[0].1)); + assert_eq!(index.lookup(EventSequence::new(10)), Some(offsets[9].1)); + } + + #[test] + fn rebuild_indexes_at_interval() { + let (sim, fd) = setup(); + let offsets = write_n_events(&sim, fd, 600); + sim.sync(fd).unwrap(); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, 256).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(600))); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + assert_eq!(index.last_seq(), Some(EventSequence::new(600))); + assert_eq!(index.entry_count(), 4); + assert_eq!(index.lookup(EventSequence::new(1)), Some(offsets[0].1)); + assert_eq!(index.lookup(EventSequence::new(257)), Some(offsets[256].1)); + assert_eq!(index.lookup(EventSequence::new(256)), Some(offsets[0].1)); + assert_eq!(index.lookup(EventSequence::new(513)), Some(offsets[512].1)); + assert_eq!(index.lookup(EventSequence::new(600)), Some(offsets[599].1)); + } + + #[test] + fn rebuild_truncates_corruption() { + let (sim, fd) = setup(); + write_n_events(&sim, fd, 5); + sim.sync(fd).unwrap(); + + let file_size_before = sim.file_size(fd).unwrap(); + + sim.write_all_at(fd, file_size_before, b"garbage_trailing_data") + .unwrap(); + sim.sync(fd).unwrap(); + let file_size_with_garbage = sim.file_size(fd).unwrap(); + assert!(file_size_with_garbage > file_size_before); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(5))); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + + let file_size_after = sim.file_size(fd).unwrap(); + assert_eq!(file_size_after, file_size_before); + } + + #[test] + fn rebuild_truncates_partial_record() { + let (sim, fd) = setup(); + write_n_events(&sim, fd, 3); + sim.sync(fd).unwrap(); + + let valid_end = sim.file_size(fd).unwrap(); + + let partial_header = [0u8; EVENT_HEADER_SIZE - 5]; + sim.write_all_at(fd, valid_end, &partial_header).unwrap(); + sim.sync(fd).unwrap(); + + let (_, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(3))); + assert_eq!(sim.file_size(fd).unwrap(), valid_end); + } + + #[test] + fn rebuild_truncates_at_non_monotonic_seq() { + let (sim, fd) = setup(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(0), EventSequence::new(1)).unwrap(); + + let event1 = test_event(1, b"first"); + let event2 = test_event(2, b"second"); + writer.append_event(&sim, &event1).unwrap(); + let offset_after_two = { + writer.append_event(&sim, &event2).unwrap(); + writer.position() + }; + writer.sync(&sim).unwrap(); + + let valid_size_before = offset_after_two.raw(); + + let regressed = ValidEvent { + seq: EventSequence::new(1), + timestamp: TimestampMicros::new(3_000_000), + did_hash: DidHash::from_did("did:plc:test3"), + event_type: EventTypeTag::COMMIT, + payload: b"regressed".to_vec(), + }; + encode_event_record(&sim, fd, offset_after_two, ®ressed).unwrap(); + sim.sync(fd).unwrap(); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(2))); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + assert_eq!(index.last_seq(), Some(EventSequence::new(2))); + assert_eq!(sim.file_size(fd).unwrap(), valid_size_before); + } + + #[test] + fn rebuild_interval_one_indexes_every_event() { + let (sim, fd) = setup(); + let offsets = write_n_events(&sim, fd, 10); + sim.sync(fd).unwrap(); + + let (index, _) = rebuild_from_segment(&sim, fd, 1).unwrap(); + assert_eq!(index.entry_count(), 10); + + offsets.iter().enumerate().for_each(|(i, (seq, offset))| { + assert_eq!(index.lookup(*seq), Some(*offset), "event {i} lookup failed"); + }); + } + + #[test] + fn rebuild_and_save_load_round_trip() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let fd = sim + .open(Path::new("/test/segment.tqe"), OpenOptions::read_write()) + .unwrap(); + write_n_events(&sim, fd, 300); + sim.sync(fd).unwrap(); + + let (index, last_seq) = rebuild_from_segment(&sim, fd, 256).unwrap(); + assert_eq!(last_seq, Some(EventSequence::new(300))); + + let index_path = Path::new("/test/00000000.tqi"); + index.save(&sim, index_path).unwrap(); + + let loaded = SegmentIndex::load(&sim, index_path).unwrap().unwrap(); + assert_eq!(loaded, index); + assert_eq!(loaded.entry_count(), index.entry_count()); + assert_eq!(loaded.first_seq(), index.first_seq()); + assert_eq!(loaded.last_seq(), index.last_seq()); + } + + #[test] + fn save_overwrites_stale_tmp() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let stale_tmp = Path::new("/test/00000000.tqi.tmp"); + let stale_fd = sim.open(stale_tmp, OpenOptions::read_write()).unwrap(); + sim.write_all_at(stale_fd, 0, b"stale_garbage_from_prior_crash_xxxxxxxxxx") + .unwrap(); + sim.sync(stale_fd).unwrap(); + sim.close(stale_fd).unwrap(); + + let mut index = SegmentIndex::new(); + index.record(EventSequence::new(1), SegmentOffset::new(5)); + + let path = Path::new("/test/00000000.tqi"); + index.save(&sim, path).unwrap(); + + let loaded = SegmentIndex::load(&sim, path).unwrap().unwrap(); + assert_eq!(loaded, index); + } + + #[test] + fn rebuild_bad_magic_returns_err() { + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let fd = sim + .open(Path::new("/test/bad.tqe"), OpenOptions::read_write()) + .unwrap(); + sim.write_all_at(fd, 0, b"NOPE\x01").unwrap(); + sim.sync(fd).unwrap(); + + let result = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL); + assert!(result.is_err()); + } + + #[test] + fn rebuild_no_truncation_when_clean() { + let (sim, fd) = setup(); + write_n_events(&sim, fd, 5); + sim.sync(fd).unwrap(); + + let size_before = sim.file_size(fd).unwrap(); + rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap(); + let size_after = sim.file_size(fd).unwrap(); + assert_eq!(size_before, size_after); + } + + #[test] + fn lookup_at_before_all_returns_none() { + let mut index = SegmentIndex::new(); + index.record(EventSequence::new(1), SegmentOffset::new(5)); + + assert_eq!(index.lookup(EventSequence::BEFORE_ALL), None); + } +} diff --git a/crates/tranquil-store/src/eventlog/types.rs b/crates/tranquil-store/src/eventlog/types.rs new file mode 100644 index 0000000..fee53e6 --- /dev/null +++ b/crates/tranquil-store/src/eventlog/types.rs @@ -0,0 +1,522 @@ +use serde::{Deserialize, Serialize}; +use tranquil_db_traits::SequenceNumber; + +pub const MAX_EVENT_PAYLOAD: u32 = 4 * 1024 * 1024; +pub const DEFAULT_SEGMENT_SIZE: u64 = 64 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct EventSequence(u64); + +impl EventSequence { + pub const BEFORE_ALL: Self = Self(0); + + pub fn new(seq: u64) -> Self { + assert!( + seq > 0, + "EventSequence must be positive; use BEFORE_ALL for cursor start" + ); + Self(seq) + } + + pub fn raw(self) -> u64 { + self.0 + } + + pub fn next(self) -> Self { + Self(self.0.checked_add(1).expect("EventSequence overflow")) + } + + pub fn prev_or_before_all(self) -> Self { + match self.0 { + 0 | 1 => Self::BEFORE_ALL, + n => Self(n - 1), + } + } + + pub fn as_i64(self) -> i64 { + i64::try_from(self.0).expect("EventSequence exceeds i64::MAX") + } + + pub fn from_i64(n: i64) -> Option { + match u64::try_from(n) { + Ok(0) | Err(_) => None, + Ok(v) => Some(Self(v)), + } + } + + pub fn cursor_from_i64(n: i64) -> Option { + u64::try_from(n).ok().map(Self) + } +} + +impl From for SequenceNumber { + fn from(es: EventSequence) -> Self { + SequenceNumber::from_raw(es.as_i64()) + } +} + +impl TryFrom for EventSequence { + type Error = &'static str; + + fn try_from(seq: SequenceNumber) -> Result { + let raw = seq.as_i64(); + match u64::try_from(raw) { + Ok(0) => Err("SequenceNumber 0 maps to BEFORE_ALL, not a valid EventSequence"), + Ok(v) => Ok(Self(v)), + Err(_) => Err("negative SequenceNumber cannot convert to EventSequence"), + } + } +} + +impl std::fmt::Display for EventSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct SegmentId(u32); + +impl SegmentId { + pub fn new(id: u32) -> Self { + Self(id) + } + + pub fn raw(self) -> u32 { + self.0 + } + + pub fn next(self) -> Self { + Self(self.0.checked_add(1).expect("SegmentId overflow")) + } +} + +impl std::fmt::Display for SegmentId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:08}", self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct SegmentOffset(u64); + +impl SegmentOffset { + pub const fn new(offset: u64) -> Self { + Self(offset) + } + + pub const fn raw(self) -> u64 { + self.0 + } + + pub fn advance(self, delta: u64) -> Self { + Self(self.0.checked_add(delta).expect("SegmentOffset overflow")) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +pub struct EventLength(u32); + +impl EventLength { + pub fn new(length: u32) -> Self { + assert!( + length <= MAX_EVENT_PAYLOAD, + "EventLength {length} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}" + ); + Self(length) + } + + pub fn raw(self) -> u32 { + self.0 + } + + pub fn as_u64(self) -> u64 { + u64::from(self.0) + } +} + +impl<'de> Deserialize<'de> for EventLength { + fn deserialize>(deserializer: D) -> Result { + let raw = u32::deserialize(deserializer)?; + if raw > MAX_EVENT_PAYLOAD { + return Err(serde::de::Error::custom(format_args!( + "EventLength {raw} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}" + ))); + } + Ok(Self(raw)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct DidHash(u32); + +impl DidHash { + pub fn from_did(did: &str) -> Self { + Self(xxhash_rust::xxh3::xxh3_64(did.as_bytes()) as u32) + } + + pub fn from_raw(hash: u32) -> Self { + Self(hash) + } + + pub fn raw(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +pub struct EventTypeTag(u8); + +impl EventTypeTag { + pub const COMMIT: Self = Self(1); + pub const IDENTITY: Self = Self(2); + pub const ACCOUNT: Self = Self(3); + pub const SYNC: Self = Self(4); + + pub fn from_raw(tag: u8) -> Option { + match tag { + 1..=4 => Some(Self(tag)), + _ => None, + } + } + + pub fn raw(self) -> u8 { + self.0 + } + + pub fn to_repo_event_type(self) -> tranquil_db_traits::RepoEventType { + match self { + Self::COMMIT => tranquil_db_traits::RepoEventType::Commit, + Self::IDENTITY => tranquil_db_traits::RepoEventType::Identity, + Self::ACCOUNT => tranquil_db_traits::RepoEventType::Account, + Self::SYNC => tranquil_db_traits::RepoEventType::Sync, + _ => unreachable!("EventTypeTag invariant guarantees valid discriminant"), + } + } +} + +impl<'de> Deserialize<'de> for EventTypeTag { + fn deserialize>(deserializer: D) -> Result { + let raw = u8::deserialize(deserializer)?; + Self::from_raw(raw) + .ok_or_else(|| serde::de::Error::custom(format_args!("invalid EventTypeTag: {raw}"))) + } +} + +impl std::fmt::Display for EventTypeTag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::COMMIT => write!(f, "Commit"), + Self::IDENTITY => write!(f, "Identity"), + Self::ACCOUNT => write!(f, "Account"), + Self::SYNC => write!(f, "Sync"), + _ => unreachable!("EventTypeTag invariant violated: raw value {}", self.0), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct TimestampMicros(u64); + +impl TimestampMicros { + pub fn new(us: u64) -> Self { + Self(us) + } + + pub fn raw(self) -> u64 { + self.0 + } + + pub fn now() -> Self { + let duration = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock before unix epoch"); + Self( + duration + .as_secs() + .saturating_mul(1_000_000) + .saturating_add(u64::from(duration.subsec_micros())), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn event_sequence_lifecycle() { + let seq = EventSequence::new(1); + assert_eq!(seq.raw(), 1); + assert_eq!(seq.next(), EventSequence::new(2)); + assert_eq!(seq.as_i64(), 1); + } + + #[test] + fn event_sequence_before_all() { + assert_eq!(EventSequence::BEFORE_ALL.raw(), 0); + } + + #[test] + fn event_sequence_prev_or_before_all() { + assert_eq!( + EventSequence::BEFORE_ALL.prev_or_before_all(), + EventSequence::BEFORE_ALL + ); + assert_eq!( + EventSequence::new(1).prev_or_before_all(), + EventSequence::BEFORE_ALL + ); + assert_eq!( + EventSequence::new(2).prev_or_before_all(), + EventSequence::new(1) + ); + assert_eq!( + EventSequence::new(100).prev_or_before_all(), + EventSequence::new(99) + ); + } + + #[test] + #[should_panic(expected = "EventSequence must be positive")] + fn event_sequence_zero_panics() { + EventSequence::new(0); + } + + #[test] + fn event_sequence_i64_round_trip() { + let seq = EventSequence::new(42); + let as_i64 = seq.as_i64(); + assert_eq!(EventSequence::from_i64(as_i64), Some(seq)); + } + + #[test] + fn event_sequence_from_i64_rejects_zero_and_negative() { + assert_eq!(EventSequence::from_i64(0), None); + assert_eq!(EventSequence::from_i64(-1), None); + } + + #[test] + fn event_sequence_cursor_from_i64_allows_zero() { + assert_eq!( + EventSequence::cursor_from_i64(0), + Some(EventSequence::BEFORE_ALL) + ); + assert_eq!( + EventSequence::cursor_from_i64(1), + Some(EventSequence::new(1)) + ); + assert_eq!(EventSequence::cursor_from_i64(-1), None); + } + + #[test] + #[should_panic(expected = "EventSequence overflow")] + fn event_sequence_overflow_panics() { + EventSequence::new(u64::MAX).next(); + } + + #[test] + fn segment_id_display_zero_padded() { + assert_eq!(SegmentId::new(0).to_string(), "00000000"); + assert_eq!(SegmentId::new(1).to_string(), "00000001"); + assert_eq!(SegmentId::new(99999999).to_string(), "99999999"); + } + + #[test] + fn segment_id_next_increments() { + assert_eq!(SegmentId::new(0).next(), SegmentId::new(1)); + assert_eq!(SegmentId::new(99).next(), SegmentId::new(100)); + } + + #[test] + #[should_panic(expected = "SegmentId overflow")] + fn segment_id_overflow_panics() { + SegmentId::new(u32::MAX).next(); + } + + #[test] + fn segment_offset_advance() { + let offset = SegmentOffset::new(100); + assert_eq!(offset.advance(50), SegmentOffset::new(150)); + } + + #[test] + #[should_panic(expected = "SegmentOffset overflow")] + fn segment_offset_overflow_panics() { + SegmentOffset::new(u64::MAX).advance(1); + } + + #[test] + fn event_length_valid() { + let len = EventLength::new(1024); + assert_eq!(len.raw(), 1024); + assert_eq!(len.as_u64(), 1024); + } + + #[test] + fn event_length_max_accepted() { + let len = EventLength::new(MAX_EVENT_PAYLOAD); + assert_eq!(len.raw(), MAX_EVENT_PAYLOAD); + } + + #[test] + #[should_panic(expected = "exceeds MAX_EVENT_PAYLOAD")] + fn event_length_overflow_panics() { + EventLength::new(MAX_EVENT_PAYLOAD + 1); + } + + #[test] + fn did_hash_deterministic() { + let hash1 = DidHash::from_did("did:plc:abc123"); + let hash2 = DidHash::from_did("did:plc:abc123"); + assert_eq!(hash1, hash2); + } + + #[test] + fn did_hash_different_dids_differ() { + let hash1 = DidHash::from_did("did:plc:abc123"); + let hash2 = DidHash::from_did("did:plc:xyz789"); + assert_ne!(hash1, hash2); + } + + #[test] + fn event_type_tag_known_variants() { + assert_eq!(EventTypeTag::COMMIT.raw(), 1); + assert_eq!(EventTypeTag::IDENTITY.raw(), 2); + assert_eq!(EventTypeTag::ACCOUNT.raw(), 3); + assert_eq!(EventTypeTag::SYNC.raw(), 4); + } + + #[test] + fn event_type_tag_from_raw_valid() { + assert_eq!(EventTypeTag::from_raw(1), Some(EventTypeTag::COMMIT)); + assert_eq!(EventTypeTag::from_raw(2), Some(EventTypeTag::IDENTITY)); + assert_eq!(EventTypeTag::from_raw(3), Some(EventTypeTag::ACCOUNT)); + assert_eq!(EventTypeTag::from_raw(4), Some(EventTypeTag::SYNC)); + } + + #[test] + fn event_type_tag_from_raw_invalid() { + assert_eq!(EventTypeTag::from_raw(0), None); + assert_eq!(EventTypeTag::from_raw(5), None); + assert_eq!(EventTypeTag::from_raw(255), None); + } + + #[test] + fn event_type_tag_display() { + assert_eq!(EventTypeTag::COMMIT.to_string(), "Commit"); + assert_eq!(EventTypeTag::IDENTITY.to_string(), "Identity"); + assert_eq!(EventTypeTag::ACCOUNT.to_string(), "Account"); + assert_eq!(EventTypeTag::SYNC.to_string(), "Sync"); + } + + #[test] + fn timestamp_micros_round_trip() { + let ts = TimestampMicros::new(1_700_000_000_000_000); + assert_eq!(ts.raw(), 1_700_000_000_000_000); + } + + #[test] + fn timestamp_micros_now_is_reasonable() { + let ts = TimestampMicros::now(); + assert!(ts.raw() > 1_700_000_000_000_000); + } + + #[test] + fn postcard_round_trip_event_sequence() { + let seq = EventSequence::new(42); + let bytes = postcard::to_allocvec(&seq).unwrap(); + let decoded: EventSequence = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(seq, decoded); + } + + #[test] + fn postcard_round_trip_segment_id() { + let id = SegmentId::new(7); + let bytes = postcard::to_allocvec(&id).unwrap(); + let decoded: SegmentId = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(id, decoded); + } + + #[test] + fn postcard_round_trip_did_hash() { + let hash = DidHash::from_did("did:plc:test"); + let bytes = postcard::to_allocvec(&hash).unwrap(); + let decoded: DidHash = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(hash, decoded); + } + + #[test] + fn postcard_round_trip_event_type_tag() { + let tag = EventTypeTag::COMMIT; + let bytes = postcard::to_allocvec(&tag).unwrap(); + let decoded: EventTypeTag = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(tag, decoded); + } + + #[test] + fn postcard_round_trip_timestamp_micros() { + let ts = TimestampMicros::new(1_700_000_000_000_000); + let bytes = postcard::to_allocvec(&ts).unwrap(); + let decoded: TimestampMicros = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(ts, decoded); + } + + #[test] + fn postcard_rejects_invalid_event_type_tag() { + let bytes = postcard::to_allocvec(&0u8).unwrap(); + assert!(postcard::from_bytes::(&bytes).is_err()); + + let bytes = postcard::to_allocvec(&5u8).unwrap(); + assert!(postcard::from_bytes::(&bytes).is_err()); + + let bytes = postcard::to_allocvec(&255u8).unwrap(); + assert!(postcard::from_bytes::(&bytes).is_err()); + } + + #[test] + fn postcard_rejects_oversized_event_length() { + let oversized = MAX_EVENT_PAYLOAD + 1; + let bytes = postcard::to_allocvec(&oversized).unwrap(); + assert!(postcard::from_bytes::(&bytes).is_err()); + } + + #[test] + fn event_sequence_to_sequence_number() { + let es = EventSequence::new(42); + let sn: SequenceNumber = es.into(); + assert_eq!(sn.as_i64(), 42); + } + + #[test] + fn event_sequence_before_all_to_sequence_number() { + let sn: SequenceNumber = EventSequence::BEFORE_ALL.into(); + assert_eq!(sn, SequenceNumber::ZERO); + } + + #[test] + fn sequence_number_to_event_sequence() { + let sn = SequenceNumber::from_raw(42); + let es = EventSequence::try_from(sn).unwrap(); + assert_eq!(es.raw(), 42); + } + + #[test] + fn sequence_number_zero_rejects_to_event_sequence() { + let result = EventSequence::try_from(SequenceNumber::ZERO); + assert!(result.is_err()); + } + + #[test] + fn sequence_number_negative_rejects_to_event_sequence() { + let result = EventSequence::try_from(SequenceNumber::from_raw(-1)); + assert!(result.is_err()); + } + + #[test] + fn postcard_accepts_max_event_length() { + let bytes = postcard::to_allocvec(&MAX_EVENT_PAYLOAD).unwrap(); + let decoded: EventLength = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(decoded.raw(), MAX_EVENT_PAYLOAD); + } +} diff --git a/crates/tranquil-store/src/eventlog/writer.rs b/crates/tranquil-store/src/eventlog/writer.rs new file mode 100644 index 0000000..1b67a5e --- /dev/null +++ b/crates/tranquil-store/src/eventlog/writer.rs @@ -0,0 +1,972 @@ +use std::io; +use std::sync::Arc; + +use tracing::warn; + +use crate::io::StorageIO; + +use super::manager::SegmentManager; +use super::segment_file::{SEGMENT_HEADER_SIZE, SegmentWriter, ValidEvent}; +use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment}; +use super::types::{ + DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset, + TimestampMicros, +}; + +#[derive(Debug)] +pub struct SyncResult { + pub synced_through: EventSequence, + pub segment_id: SegmentId, + pub position: SegmentOffset, + pub flushed_events: Vec, +} + +pub struct EventLogWriter { + manager: Arc>, + active_writer: SegmentWriter, + active_index: SegmentIndex, + next_seq: EventSequence, + synced_seq: EventSequence, + index_interval: usize, + event_count_in_segment: usize, + last_event_offset: Option, + pending_events: Vec, +} + +impl EventLogWriter { + pub fn open(manager: Arc>, index_interval: usize) -> io::Result { + assert!(index_interval > 0, "index_interval must be positive"); + + let segments = manager.list_segments()?; + + match segments.last() { + None => Self::init_fresh( + manager, + SegmentId::new(1), + EventSequence::new(1), + index_interval, + ), + Some(&last_id) => Self::recover_active(manager, &segments, last_id, index_interval), + } + } + + fn init_fresh( + manager: Arc>, + segment_id: SegmentId, + next_seq: EventSequence, + index_interval: usize, + ) -> io::Result { + let fd = manager.open_for_append(segment_id)?; + manager.io().truncate(fd, 0)?; + let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq)?; + writer.sync(manager.io())?; + manager.io().sync_dir(manager.segments_dir())?; + + Ok(Self { + manager, + active_writer: writer, + active_index: SegmentIndex::new(), + next_seq, + synced_seq: next_seq.prev_or_before_all(), + index_interval, + event_count_in_segment: 0, + last_event_offset: None, + pending_events: Vec::new(), + }) + } + + fn recover_active( + manager: Arc>, + segments: &[SegmentId], + active_id: SegmentId, + index_interval: usize, + ) -> io::Result { + let fd = manager.open_for_append(active_id)?; + + let (index, last_seq_in_active) = match rebuild_from_segment( + manager.io(), + fd, + index_interval, + ) { + Ok(result) => result, + Err(rebuild_err) => { + let file_size = manager.io().file_size(fd)?; + if file_size <= SEGMENT_HEADER_SIZE as u64 { + manager.io().truncate(fd, 0)?; + let prev_segments = &segments[..segments.len().saturating_sub(1)]; + let next_seq = find_last_seq_from_segments(&manager, prev_segments)? + .map_or(EventSequence::new(1), |s| s.next()); + return Self::init_fresh( + Arc::clone(&manager), + active_id, + next_seq, + index_interval, + ); + } + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "segment {active_id} rebuild failed ({file_size} bytes on disk): {rebuild_err}" + ), + )); + } + }; + + let position = SegmentOffset::new(manager.io().file_size(fd)?); + + let prev_segments = &segments[..segments.len().saturating_sub(1)]; + + let next_seq = match last_seq_in_active { + Some(seq) => { + if let Some(sealed_last) = find_last_seq_from_segments(&manager, prev_segments)? + && seq <= sealed_last + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "active segment last seq ({seq}) must exceed sealed segments' \ + last seq ({sealed_last}): cross-segment corruption detected" + ), + )); + } + seq.next() + } + None => find_last_seq_from_segments(&manager, prev_segments)? + .map_or(EventSequence::new(1), |s| s.next()), + }; + + let synced_seq = next_seq.prev_or_before_all(); + + let event_count_in_segment = match (index.first_seq(), index.last_seq()) { + (Some(first), Some(last)) => { + debug_assert!( + first <= last, + "index invariant violated: first_seq {first} > last_seq {last}" + ); + usize::try_from(last.raw() - first.raw() + 1).expect("event count exceeds usize") + } + _ => 0, + }; + + let base_seq = index.first_seq().unwrap_or(next_seq); + + let last_event_offset = index.last_seq().and_then(|seq| index.lookup(seq)); + + let writer = SegmentWriter::resume( + manager.io(), + fd, + active_id, + position, + base_seq, + last_seq_in_active, + ); + + if let Err(e) = manager.io().delete(&manager.index_path(active_id)) + && e.kind() != io::ErrorKind::NotFound + { + warn!(segment = %active_id, error = %e, "failed to delete stale index"); + } + + Ok(Self { + manager, + active_writer: writer, + active_index: index, + next_seq, + synced_seq, + index_interval, + event_count_in_segment, + last_event_offset, + pending_events: Vec::new(), + }) + } + + pub fn append( + &mut self, + did_hash: DidHash, + event_type: EventTypeTag, + payload: Vec, + ) -> io::Result { + let payload_len = u32::try_from(payload.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "payload exceeds u32::MAX"))?; + if payload_len > MAX_EVENT_PAYLOAD { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "payload length {payload_len} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}" + ), + )); + } + + let seq = self.next_seq; + let timestamp = TimestampMicros::now(); + + let event = ValidEvent { + seq, + timestamp, + did_hash, + event_type, + payload, + }; + + let offset = self.active_writer.append_event(self.manager.io(), &event)?; + + let should_index = self.event_count_in_segment == 0 + || self + .event_count_in_segment + .is_multiple_of(self.index_interval); + if should_index { + self.active_index.record(seq, offset); + } + + self.event_count_in_segment = self + .event_count_in_segment + .checked_add(1) + .expect("event_count_in_segment overflow"); + self.last_event_offset = Some(offset); + self.next_seq = seq.next(); + self.pending_events.push(event); + + Ok(seq) + } + + pub fn sync(&mut self) -> io::Result { + if !self.pending_events.is_empty() { + self.active_writer.sync(self.manager.io())?; + } + + let flushed = std::mem::take(&mut self.pending_events); + self.synced_seq = flushed.last().map(|e| e.seq).unwrap_or(self.synced_seq); + + Ok(SyncResult { + synced_through: self.synced_seq, + segment_id: self.active_writer.segment_id(), + position: self.active_writer.position(), + flushed_events: flushed, + }) + } + + pub fn rotate_if_needed(&mut self) -> io::Result> { + if !self.manager.should_rotate(self.active_writer.position()) { + return Ok(None); + } + + if !self.pending_events.is_empty() { + return Ok(None); + } + + let old_id = self.active_writer.segment_id(); + + self.ensure_last_event_indexed(); + + self.manager.seal_segment(old_id, &self.active_index)?; + + let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?; + + match SegmentWriter::new::(self.manager.io(), new_fd, new_id, self.next_seq) { + Ok(writer) => { + self.active_writer = writer; + self.active_index = SegmentIndex::new(); + self.event_count_in_segment = 0; + self.last_event_offset = None; + self.manager.commit_rotation(new_id, new_fd); + Ok(Some(old_id)) + } + Err(e) => { + self.manager.rollback_rotation(new_id, new_fd); + Err(e) + } + } + } + + pub fn checkpoint_index(&self) -> io::Result<()> { + if self.active_index.entry_count() == 0 { + return Ok(()); + } + let path = self.manager.index_path(self.active_writer.segment_id()); + self.active_index.save(self.manager.io(), &path) + } + + pub fn current_seq(&self) -> EventSequence { + self.next_seq.prev_or_before_all() + } + + pub fn synced_seq(&self) -> EventSequence { + self.synced_seq + } + + pub fn active_segment_id(&self) -> SegmentId { + self.active_writer.segment_id() + } + + pub fn active_index_snapshot(&self) -> SegmentIndex { + self.active_index.clone() + } + + pub fn position(&self) -> SegmentOffset { + self.active_writer.position() + } + + pub fn shutdown(&mut self) -> io::Result<()> { + let _ = self.sync()?; + self.ensure_last_event_indexed(); + self.checkpoint_index() + } + + fn ensure_last_event_indexed(&mut self) { + let last_written = self.next_seq.prev_or_before_all(); + let needs_final_index = self.last_event_offset.is_some() + && (self.active_index.last_seq() != Some(last_written)); + if let (true, Some(offset)) = (needs_final_index, self.last_event_offset) { + self.active_index.record(last_written, offset); + } + } +} + +fn find_last_seq_from_segments( + manager: &SegmentManager, + segments: &[SegmentId], +) -> io::Result> { + segments.iter().rev().try_fold(None, |acc, &seg_id| { + if acc.is_some() { + return Ok(acc); + } + + match SegmentIndex::load(manager.io(), &manager.index_path(seg_id)) { + Ok(Some(idx)) => Ok(idx.last_seq()), + Err(e) if e.kind() != io::ErrorKind::InvalidData => Err(e), + _ => { + let fd = manager.open_for_read(seg_id)?; + let (_, last_seq) = rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL)?; + Ok(last_seq) + } + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::segment_file::{EVENT_RECORD_OVERHEAD, SegmentReader}; + use crate::eventlog::segment_index::DEFAULT_INDEX_INTERVAL; + use crate::sim::SimulatedIO; + use std::path::{Path, PathBuf}; + + fn setup_manager(max_segment_size: u64) -> Arc> { + let sim = SimulatedIO::pristine(42); + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()) + } + + fn append_test_event( + writer: &mut EventLogWriter, + did_seed: &str, + ) -> EventSequence { + writer + .append( + DidHash::from_did(did_seed), + EventTypeTag::COMMIT, + format!("payload-{did_seed}").into_bytes(), + ) + .unwrap() + } + + #[test] + fn open_fresh_creates_segment() { + let mgr = setup_manager(64 * 1024); + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + assert_eq!(writer.active_segment_id(), SegmentId::new(1)); + assert_eq!(writer.current_seq(), EventSequence::BEFORE_ALL); + assert_eq!(writer.synced_seq(), EventSequence::BEFORE_ALL); + assert_eq!( + writer.position(), + SegmentOffset::new(SEGMENT_HEADER_SIZE as u64) + ); + + let segments = mgr.list_segments().unwrap(); + assert_eq!(segments, vec![SegmentId::new(1)]); + } + + #[test] + fn append_assigns_contiguous_sequences() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + let seqs: Vec = (1..=5) + .map(|i| append_test_event(&mut writer, &format!("did:plc:user{i}"))) + .collect(); + + assert_eq!(seqs, (1..=5).map(EventSequence::new).collect::>()); + assert_eq!(writer.current_seq(), EventSequence::new(5)); + } + + #[test] + fn sync_returns_flushed_events() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=3).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + + let result = writer.sync().unwrap(); + assert_eq!(result.synced_through, EventSequence::new(3)); + assert_eq!(result.flushed_events.len(), 3); + assert_eq!(result.segment_id, SegmentId::new(1)); + + result + .flushed_events + .iter() + .enumerate() + .for_each(|(i, event)| { + assert_eq!(event.seq, EventSequence::new(i as u64 + 1)); + }); + + assert_eq!(writer.synced_seq(), EventSequence::new(3)); + } + + #[test] + fn sync_without_pending_is_noop() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + let result = writer.sync().unwrap(); + assert_eq!(result.synced_through, EventSequence::BEFORE_ALL); + assert!(result.flushed_events.is_empty()); + } + + #[test] + fn second_sync_returns_only_new_events() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=3).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + writer.sync().unwrap(); + + (4..=5).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + let result = writer.sync().unwrap(); + assert_eq!(result.synced_through, EventSequence::new(5)); + assert_eq!(result.flushed_events.len(), 2); + assert_eq!(result.flushed_events[0].seq, EventSequence::new(4)); + assert_eq!(result.flushed_events[1].seq, EventSequence::new(5)); + } + + #[test] + fn recovery_preserves_synced_events() { + let mgr = setup_manager(64 * 1024); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=5).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + writer.sync().unwrap(); + } + + mgr.shutdown(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.current_seq(), EventSequence::new(5)); + assert_eq!(writer.synced_seq(), EventSequence::new(5)); + assert_eq!(writer.active_segment_id(), SegmentId::new(1)); + + let fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + let events = SegmentReader::open(mgr.io(), fd) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!(events.len(), 5); + } + + #[test] + fn recovery_loses_unsynced_events() { + let mgr = setup_manager(64 * 1024); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=3).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + writer.sync().unwrap(); + mgr.io().sync_dir(Path::new("/segments")).unwrap(); + + (4..=6).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + } + + mgr.shutdown(); + mgr.io().crash(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.current_seq(), EventSequence::new(3)); + assert_eq!(writer.next_seq, EventSequence::new(4)); + } + + #[test] + fn rotation_creates_new_segment() { + let payload_size = 100; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 3; + + let mgr = setup_manager(max_segment_size as u64); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=3).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xAA; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + assert!(writer.rotate_if_needed().unwrap().is_some()); + + assert_eq!(writer.active_segment_id(), SegmentId::new(2)); + assert_eq!( + writer.position(), + SegmentOffset::new(SEGMENT_HEADER_SIZE as u64) + ); + + let segments = mgr.list_segments().unwrap(); + assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(2)]); + } + + #[test] + fn rotation_seals_old_segment() { + let payload_size = 100; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2; + + let mgr = setup_manager(max_segment_size as u64); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=2).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xBB; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + + assert!(mgr.is_sealed(SegmentId::new(1))); + + let index = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1))) + .unwrap() + .unwrap(); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + assert_eq!(index.last_seq(), Some(EventSequence::new(2))); + } + + #[test] + fn sequences_continue_across_rotation() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2; + + let mgr = setup_manager(max_segment_size as u64); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=2).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xCC; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + + let seq = writer + .append( + DidHash::from_did("did:plc:user3"), + EventTypeTag::COMMIT, + vec![0xCC; payload_size], + ) + .unwrap(); + assert_eq!(seq, EventSequence::new(3)); + } + + #[test] + fn recovery_after_rotation() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2; + + let mgr = setup_manager(max_segment_size as u64); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=2).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xDD; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + + writer + .append( + DidHash::from_did("did:plc:user3"), + EventTypeTag::COMMIT, + vec![0xDD; payload_size], + ) + .unwrap(); + writer.sync().unwrap(); + } + + mgr.shutdown(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.active_segment_id(), SegmentId::new(2)); + assert_eq!(writer.current_seq(), EventSequence::new(3)); + assert_eq!(writer.next_seq, EventSequence::new(4)); + } + + #[test] + fn recovery_sealed_last_segment() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2; + + let mgr = setup_manager(max_segment_size as u64); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=2).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xEE; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + + mgr.shutdown(); + mgr.io().crash(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.next_seq, EventSequence::new(3)); + } + + #[test] + fn recovery_empty_active_after_rotation() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2; + + let mgr = setup_manager(max_segment_size as u64); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=2).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xEE; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + + mgr.shutdown(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.next_seq, EventSequence::new(3)); + + let fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + let events = SegmentReader::open(mgr.io(), fd) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!(events.len(), 2); + } + + #[test] + fn checkpoint_creates_index_file() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=10).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + writer.sync().unwrap(); + + writer.checkpoint_index().unwrap(); + + let wip = mgr.index_path(SegmentId::new(1)); + let loaded = SegmentIndex::load(mgr.io(), &wip).unwrap(); + assert!(loaded.is_some()); + } + + #[test] + fn checkpoint_empty_index_is_noop() { + let mgr = setup_manager(64 * 1024); + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + writer.checkpoint_index().unwrap(); + + let wip = mgr.index_path(SegmentId::new(1)); + let loaded = SegmentIndex::load(mgr.io(), &wip).unwrap(); + assert!(loaded.is_none()); + } + + #[test] + fn current_seq_and_synced_seq_diverge_before_sync() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + append_test_event(&mut writer, "did:plc:user1"); + append_test_event(&mut writer, "did:plc:user2"); + + assert_eq!(writer.current_seq(), EventSequence::new(2)); + assert_eq!(writer.synced_seq(), EventSequence::BEFORE_ALL); + + writer.sync().unwrap(); + + assert_eq!(writer.current_seq(), EventSequence::new(2)); + assert_eq!(writer.synced_seq(), EventSequence::new(2)); + } + + #[test] + fn sparse_index_built_at_intervals() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 4).unwrap(); + + (1..=10).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + writer.sync().unwrap(); + + assert_eq!(writer.active_index.first_seq(), Some(EventSequence::new(1))); + assert!(writer.active_index.entry_count() >= 3); + assert!(writer.active_index.lookup(EventSequence::new(1)).is_some()); + assert!(writer.active_index.lookup(EventSequence::new(5)).is_some()); + } + + #[test] + fn multi_rotation_and_recovery() { + let payload_size = 30; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 3; + + let mgr = setup_manager(max_segment_size as u64); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=9).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![i as u8; payload_size], + ) + .unwrap(); + + if i % 3 == 0 { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + writer.sync().unwrap(); + } + + mgr.shutdown(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.next_seq, EventSequence::new(10)); + + let segments = mgr.list_segments().unwrap(); + assert!(segments.len() >= 3); + } + + #[test] + fn shutdown_syncs_and_checkpoints() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + (1..=5).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + + assert_eq!(writer.synced_seq(), EventSequence::BEFORE_ALL); + + writer.shutdown().unwrap(); + + assert_eq!(writer.synced_seq(), EventSequence::new(5)); + + let wip = mgr.index_path(SegmentId::new(1)); + assert!(SegmentIndex::load(mgr.io(), &wip).unwrap().is_some()); + } + + #[test] + fn rotation_indexes_last_event() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 5; + + let mgr = setup_manager(max_segment_size as u64); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + + (1..=5).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xFF; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + + let index = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1))) + .unwrap() + .unwrap(); + + assert_eq!(index.last_seq(), Some(EventSequence::new(5))); + assert!(index.lookup(EventSequence::new(5)).is_some()); + } + + #[test] + fn open_idempotent_on_fresh() { + let mgr = setup_manager(64 * 1024); + + { + let _writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + } + mgr.shutdown(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.active_segment_id(), SegmentId::new(1)); + assert_eq!(writer.current_seq(), EventSequence::BEFORE_ALL); + } + + #[test] + fn append_after_recovery_continues_sequence() { + let mgr = setup_manager(64 * 1024); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=3).for_each(|i| { + append_test_event(&mut writer, &format!("did:plc:user{i}")); + }); + writer.sync().unwrap(); + } + + mgr.shutdown(); + + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + let seq = append_test_event(&mut writer, "did:plc:user4"); + assert_eq!(seq, EventSequence::new(4)); + writer.sync().unwrap(); + + let fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + let events = SegmentReader::open(mgr.io(), fd) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!(events.len(), 4); + assert_eq!(events[3].seq, EventSequence::new(4)); + } + + #[test] + fn recovery_falls_back_to_scan_when_index_corrupt() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2; + + let mgr = setup_manager(max_segment_size as u64); + + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + (1..=2).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xAA; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + + (3..=4).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:user{i}")), + EventTypeTag::COMMIT, + vec![0xAA; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + + mgr.shutdown(); + + let index_path = mgr.index_path(SegmentId::new(1)); + let fd = mgr + .io() + .open(&index_path, crate::OpenOptions::read_write()) + .unwrap(); + mgr.io().write_all_at(fd, 0, b"CORRUPT_GARBAGE").unwrap(); + mgr.io().sync(fd).unwrap(); + mgr.io().close(fd).unwrap(); + + let index_path_2 = mgr.index_path(SegmentId::new(2)); + let fd2 = mgr + .io() + .open(&index_path_2, crate::OpenOptions::read_write()) + .unwrap(); + mgr.io().write_all_at(fd2, 0, b"CORRUPT_GARBAGE").unwrap(); + mgr.io().sync(fd2).unwrap(); + mgr.io().close(fd2).unwrap(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + assert_eq!(writer.next_seq, EventSequence::new(5)); + } + + #[test] + fn rotation_not_needed_returns_false() { + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap(); + + append_test_event(&mut writer, "did:plc:user1"); + writer.sync().unwrap(); + + assert!(writer.rotate_if_needed().unwrap().is_none()); + } +} diff --git a/crates/tranquil-store/src/fsync_order.rs b/crates/tranquil-store/src/fsync_order.rs new file mode 100644 index 0000000..2855673 --- /dev/null +++ b/crates/tranquil-store/src/fsync_order.rs @@ -0,0 +1,7 @@ +use std::io; + +use crate::blockstore::BlocksSynced; + +pub trait PostBlockstoreHook: Send + Sync { + fn on_blocks_synced(&self, proof: &BlocksSynced) -> io::Result<()>; +} diff --git a/crates/tranquil-store/src/lib.rs b/crates/tranquil-store/src/lib.rs index 8d62cd7..349daac 100644 --- a/crates/tranquil-store/src/lib.rs +++ b/crates/tranquil-store/src/lib.rs @@ -1,4 +1,6 @@ pub mod blockstore; +pub mod eventlog; +pub mod fsync_order; mod harness; mod io; mod record; @@ -16,4 +18,4 @@ pub use record::{ FILE_MAGIC, FORMAT_VERSION, HEADER_SIZE, MAX_RECORD_PAYLOAD, RECORD_OVERHEAD, ReadRecord, RecordReader, RecordWriter, }; -pub use sim::{FaultConfig, SimulatedIO}; +pub use sim::{FaultConfig, OpRecord, SimulatedIO}; diff --git a/crates/tranquil-store/tests/eventlog_crash.rs b/crates/tranquil-store/tests/eventlog_crash.rs new file mode 100644 index 0000000..32c4754 --- /dev/null +++ b/crates/tranquil-store/tests/eventlog_crash.rs @@ -0,0 +1,642 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tranquil_store::eventlog::{ + DidHash, EVENT_RECORD_OVERHEAD, EventLogWriter, EventSequence, EventTypeTag, + SEGMENT_HEADER_SIZE, SegmentId, SegmentManager, SegmentReader, SegmentWriter, TimestampMicros, + ValidEvent, rebuild_from_segment, +}; +use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO}; + +fn setup_manager(sim: SimulatedIO, max_segment_size: u64) -> Arc> { + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()) +} + +fn append_test_event(writer: &mut EventLogWriter, seq_hint: u64) -> EventSequence { + writer + .append( + DidHash::from_did(&format!("did:plc:crash{seq_hint}")), + EventTypeTag::COMMIT, + format!("payload-{seq_hint}").into_bytes(), + ) + .unwrap() +} + +#[test] +fn synced_events_survive_crash() { + (0..500u64).for_each(|seed| { + let sim = SimulatedIO::pristine(seed); + let mgr = setup_manager(sim, 64 * 1024); + + let n = 10u64; + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=n).for_each(|i| { + append_test_event(&mut writer, i); + }); + writer.sync().unwrap(); + mgr.io().sync_dir(Path::new("/segments")).unwrap(); + } + + mgr.shutdown(); + mgr.io().crash(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + assert_eq!( + writer.synced_seq(), + EventSequence::new(n), + "seed {seed}: expected all synced events to survive" + ); + + let fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + let events = SegmentReader::open(mgr.io(), fd) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!(events.len(), n as usize, "seed {seed}"); + + events.iter().enumerate().for_each(|(i, e)| { + assert_eq!(e.seq, EventSequence::new(i as u64 + 1)); + }); + }); +} + +#[test] +fn unsynced_events_lost_on_crash() { + (0..500u64).for_each(|seed| { + let sim = SimulatedIO::pristine(seed); + let mgr = setup_manager(sim, 64 * 1024); + + let synced_count = 5u64; + let unsynced_count = 5u64; + { + let mut writer = + EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=synced_count).for_each(|i| { + append_test_event(&mut writer, i); + }); + writer.sync().unwrap(); + mgr.io().sync_dir(Path::new("/segments")).unwrap(); + + (synced_count + 1..=synced_count + unsynced_count).for_each(|i| { + append_test_event(&mut writer, i); + }); + } + + mgr.shutdown(); + mgr.io().crash(); + + let writer = + EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + let recovered_count = writer.synced_seq().raw(); + assert_eq!( + recovered_count, synced_count, + "seed {seed}: pristine IO should recover exactly {synced_count} synced events, got {recovered_count}" + ); + }); +} + +#[test] +fn sequence_monotonicity_after_recovery() { + (0..500u64).for_each(|seed| { + let sim = SimulatedIO::new(seed, FaultConfig::moderate()); + let mgr = setup_manager(sim, 64 * 1024); + + let crash_point = (seed % 15) + 3; + let write_result: Result<(), std::io::Error> = (|| { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256)?; + (1..=crash_point).try_for_each(|i| -> std::io::Result<()> { + writer.append( + DidHash::from_did(&format!("did:plc:mono{i}")), + EventTypeTag::COMMIT, + format!("data-{i}").into_bytes(), + )?; + if i % 3 == 0 { + writer.sync()?; + mgr.io().sync_dir(Path::new("/segments"))?; + } + Ok(()) + })?; + Ok(()) + })(); + let _ = write_result; + + mgr.shutdown(); + mgr.io().crash(); + + let mgr_clone = Arc::clone(&mgr); + let recovery_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut writer = EventLogWriter::open(Arc::clone(&mgr_clone), 256)?; + let new_seqs: Vec = (0..5u64) + .filter_map(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:post{i}")), + EventTypeTag::COMMIT, + format!("post-recovery-{i}").into_bytes(), + ) + .ok() + }) + .collect(); + Ok::<_, std::io::Error>(new_seqs) + })); + + let Ok(Ok(new_seqs)) = recovery_result else { + return; + }; + + new_seqs.windows(2).for_each(|pair| { + assert!( + pair[1].raw() == pair[0].raw() + 1, + "seed {seed}: non-contiguous seqs {} -> {}", + pair[0], + pair[1], + ); + }); + + if let Some(first_new) = new_seqs.first() { + assert!(first_new.raw() > 0, "seed {seed}: new sequence starts at 0"); + } + }); +} + +#[test] +fn partial_event_truncated_on_recovery() { + (0..500u64).for_each(|seed| { + let sim = SimulatedIO::pristine(seed); + let mgr = setup_manager(sim, 64 * 1024); + + let complete_count = 5u64; + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=complete_count).for_each(|i| { + append_test_event(&mut writer, i); + }); + writer.sync().unwrap(); + mgr.io().sync_dir(Path::new("/segments")).unwrap(); + } + + let fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + let file_size = mgr.io().file_size(fd).unwrap(); + let partial_bytes = ((seed % 20) + 1) as usize; + let junk: Vec = (0..partial_bytes) + .map(|i| (i as u8).wrapping_add(seed as u8)) + .collect(); + mgr.io().write_all_at(fd, file_size, &junk).unwrap(); + mgr.io().sync(fd).unwrap(); + + mgr.shutdown(); + mgr.io().crash(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + assert_eq!( + writer.synced_seq(), + EventSequence::new(complete_count), + "seed {seed}: partial write should be truncated, preserving {complete_count} events" + ); + }); +} + +#[test] +fn cross_segment_recovery() { + (0..200u64).for_each(|seed| { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let events_per_segment = 3; + let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64; + + let sim = SimulatedIO::pristine(seed); + let mgr = setup_manager(sim, max_segment_size); + + let sealed_events = 9u64; + let trailing_unsynced = 2u64; + let total_events = sealed_events + trailing_unsynced; + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=total_events).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:xseg{i}")), + EventTypeTag::COMMIT, + vec![i as u8; payload_size], + ) + .unwrap(); + + if i % events_per_segment as u64 == 0 && i <= sealed_events { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + mgr.io().sync_dir(Path::new("/segments")).unwrap(); + } + + mgr.shutdown(); + mgr.io().crash(); + + let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + let recovered = writer.synced_seq().raw(); + + let sealed_segments = mgr.list_segments().unwrap(); + let sealed_count = sealed_segments.len().saturating_sub(1); + + assert!( + recovered >= (sealed_count as u64) * events_per_segment as u64, + "seed {seed}: recovered {recovered} but expected at least {} sealed events", + sealed_count * events_per_segment, + ); + + sealed_segments[..sealed_count].iter().for_each(|&seg_id| { + let fd = mgr.open_for_read(seg_id).unwrap(); + let events = SegmentReader::open(mgr.io(), fd) + .unwrap() + .valid_prefix() + .unwrap(); + assert_eq!( + events.len(), + events_per_segment, + "seed {seed}: sealed segment {seg_id} should have {events_per_segment} events" + ); + }); + }); +} + +#[test] +fn corrupt_index_triggers_rebuild() { + (0..200u64).for_each(|seed| { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * 3) as u64; + + let sim = SimulatedIO::pristine(seed); + let mgr = setup_manager(sim, max_segment_size); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=6).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:idx{i}")), + EventTypeTag::COMMIT, + vec![0xAA; payload_size], + ) + .unwrap(); + if i % 3 == 0 { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + writer.sync().unwrap(); + } + mgr.shutdown(); + + let index_path = mgr.index_path(SegmentId::new(1)); + if let Ok(fd) = mgr.io().open(&index_path, OpenOptions::read_write()) { + mgr.io() + .write_all_at(fd, 0, b"CORRUPT_INDEX_GARBAGE_DATA_XYZ") + .unwrap(); + mgr.io().sync(fd).unwrap(); + mgr.io().close(fd).unwrap(); + } + + let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + + assert!( + writer.synced_seq().raw() >= 6, + "seed {seed}: recovery after corrupt index should find all events, got seq {}", + writer.synced_seq(), + ); + }); +} + +#[test] +fn large_sealed_segment_index_rebuild_latency() { + let payload_size = 1024; + let event_count = 64_000u64; + + let sim = SimulatedIO::pristine(42); + let mgr = setup_manager(sim, 256 * 1024 * 1024); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=event_count).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:bench{i}")), + EventTypeTag::COMMIT, + vec![0xBB; payload_size], + ) + .unwrap(); + }); + writer.sync().unwrap(); + writer.checkpoint_index().unwrap(); + } + mgr.shutdown(); + + let index_path = mgr.index_path(SegmentId::new(1)); + let _ = mgr.io().delete(&index_path); + + let fd = mgr.open_for_read(SegmentId::new(1)).unwrap(); + + let start = std::time::Instant::now(); + let (index, last_seq) = rebuild_from_segment(mgr.io(), fd, 256).unwrap(); + let elapsed = start.elapsed(); + + assert_eq!(last_seq, Some(EventSequence::new(event_count))); + assert!(index.entry_count() > 0); + assert!( + elapsed.as_secs() < 2, + "index rebuild took {:?}, exceeds 2s budget", + elapsed, + ); +} + +#[test] +fn corrupt_metadata_triggers_scan() { + let sim = SimulatedIO::pristine(42); + let mgr = setup_manager(sim, 64 * 1024); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=10).for_each(|i| { + append_test_event(&mut writer, i); + }); + writer.sync().unwrap(); + writer.checkpoint_index().unwrap(); + } + mgr.shutdown(); + + let index_path = mgr.index_path(SegmentId::new(1)); + if let Ok(fd) = mgr.io().open(&index_path, OpenOptions::read_write()) { + mgr.io() + .write_all_at(fd, 0, b"TOTALLY_CORRUPT_META") + .unwrap(); + mgr.io().sync(fd).unwrap(); + mgr.io().close(fd).unwrap(); + } + + let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + assert_eq!( + writer.synced_seq(), + EventSequence::new(10), + "recovery via segment scan should find all 10 events" + ); +} + +#[test] +fn pristine_comparison_under_faults() { + (0..500u64).for_each(|seed| { + let event_count = 15u64; + let sync_interval = 5u64; + + let pristine_sim = SimulatedIO::pristine(seed); + let pristine_mgr = setup_manager(pristine_sim, 64 * 1024); + + { + let mut writer = EventLogWriter::open(Arc::clone(&pristine_mgr), 256).unwrap(); + (1..=event_count).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:prist{i}")), + EventTypeTag::COMMIT, + format!("pristine-{i}").into_bytes(), + ) + .unwrap(); + if i % sync_interval == 0 { + writer.sync().unwrap(); + } + }); + writer.sync().unwrap(); + } + pristine_mgr.shutdown(); + + let pristine_fd = pristine_mgr.open_for_read(SegmentId::new(1)).unwrap(); + let pristine_events = SegmentReader::open(pristine_mgr.io(), pristine_fd) + .unwrap() + .valid_prefix() + .unwrap(); + + let faulty_sim = SimulatedIO::new(seed, FaultConfig::moderate()); + let faulty_mgr = setup_manager(faulty_sim, 64 * 1024); + + let write_ok = (|| -> std::io::Result<()> { + let mut writer = EventLogWriter::open(Arc::clone(&faulty_mgr), 256)?; + (1..=event_count).try_for_each(|i| -> std::io::Result<()> { + writer.append( + DidHash::from_did(&format!("did:plc:prist{i}")), + EventTypeTag::COMMIT, + format!("pristine-{i}").into_bytes(), + )?; + if i % sync_interval == 0 { + let _ = writer.sync(); + let _ = faulty_mgr.io().sync_dir(Path::new("/segments")); + } + Ok(()) + })?; + let _ = writer.sync(); + Ok(()) + })(); + let _ = write_ok; + + faulty_mgr.shutdown(); + faulty_mgr.io().crash(); + + let faulty_clone = Arc::clone(&faulty_mgr); + let recovery = std::panic::catch_unwind(std::panic::AssertUnwindSafe( + || -> std::io::Result>> { + let recovered_writer = EventLogWriter::open(Arc::clone(&faulty_clone), 256)?; + + let recovered_seq = recovered_writer.synced_seq().raw(); + assert!( + recovered_seq <= event_count, + "seed {seed}: recovered {recovered_seq} > written {event_count}" + ); + + if recovered_seq == 0 { + return Ok(None); + } + + let fd = faulty_clone.open_for_read(SegmentId::new(1))?; + let events = SegmentReader::open(faulty_clone.io(), fd)?.valid_prefix()?; + Ok(Some(events)) + }, + )); + + if let Ok(Ok(Some(recovered_events))) = recovery { + let is_prefix = recovered_events + .iter() + .zip(pristine_events.iter()) + .all(|(r, p)| r.seq == p.seq && r.payload == p.payload); + + assert!( + is_prefix, + "seed {seed}: recovered events must be a prefix of pristine" + ); + } + }); +} + +#[test] +fn bit_flip_detected_by_checksum() { + (0..1000u64).for_each(|seed| { + let sim = SimulatedIO::pristine(seed); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let fd = sim + .open(Path::new("/test/segment.tqe"), OpenOptions::read_write()) + .unwrap(); + let mut writer = + SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap(); + + let data_len = ((seed % 256) as usize).max(1); + let event = ValidEvent { + seq: EventSequence::new(1), + timestamp: TimestampMicros::new(1_000_000), + did_hash: DidHash::from_did("did:plc:bitflip"), + event_type: EventTypeTag::COMMIT, + payload: vec![0xAA; data_len], + }; + writer.append_event(&sim, &event).unwrap(); + writer.sync(&sim).unwrap(); + + let record_start = SEGMENT_HEADER_SIZE as u64; + let record_end = record_start + EVENT_RECORD_OVERHEAD as u64 + data_len as u64; + let flip_pos = record_start + (seed.wrapping_mul(7) % (record_end - record_start)); + let flip_bit = (seed.wrapping_mul(13) % 8) as u8; + + let mut byte_buf = [0u8; 1]; + sim.read_exact_at(fd, flip_pos, &mut byte_buf).unwrap(); + byte_buf[0] ^= 1 << flip_bit; + sim.write_all_at(fd, flip_pos, &byte_buf).unwrap(); + + use tranquil_store::eventlog::ReadEventRecord; + let mut reader = SegmentReader::open(&sim, fd).unwrap(); + let record = reader.next().unwrap().unwrap(); + assert!( + !matches!(record, ReadEventRecord::Valid { .. }), + "seed {seed}: bit flip at offset {flip_pos} bit {flip_bit} was not detected" + ); + }); +} + +fn fault_configs() -> Vec<(&'static str, FaultConfig)> { + vec![ + ( + "partial_writes_only", + FaultConfig { + partial_write_probability: 0.15, + ..FaultConfig::none() + }, + ), + ( + "sync_failures_only", + FaultConfig { + sync_failure_probability: 0.10, + dir_sync_failure_probability: 0.05, + ..FaultConfig::none() + }, + ), + ("combined", FaultConfig::moderate()), + ( + "bit_flips_only", + FaultConfig { + bit_flip_on_read_probability: 0.05, + ..FaultConfig::none() + }, + ), + ] +} + +#[test] +fn pristine_comparison_parameterized_faults() { + fault_configs().iter().for_each(|(config_name, config)| { + (0..200u64).for_each(|seed| { + let event_count = 10u64; + + let pristine_sim = SimulatedIO::pristine(seed); + let pristine_mgr = setup_manager(pristine_sim, 64 * 1024); + { + let mut writer = + EventLogWriter::open(Arc::clone(&pristine_mgr), 256).unwrap(); + (1..=event_count).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:param{i}")), + EventTypeTag::COMMIT, + format!("param-{i}").into_bytes(), + ) + .unwrap(); + if i % 4 == 0 { + writer.sync().unwrap(); + } + }); + writer.sync().unwrap(); + } + pristine_mgr.shutdown(); + + let pristine_fd = pristine_mgr.open_for_read(SegmentId::new(1)).unwrap(); + let pristine_events = SegmentReader::open(pristine_mgr.io(), pristine_fd) + .unwrap() + .valid_prefix() + .unwrap(); + + let faulty_sim = SimulatedIO::new(seed, *config); + let faulty_mgr = setup_manager(faulty_sim, 64 * 1024); + let _ = (|| -> std::io::Result<()> { + let mut writer = + EventLogWriter::open(Arc::clone(&faulty_mgr), 256)?; + (1..=event_count).try_for_each(|i| -> std::io::Result<()> { + writer.append( + DidHash::from_did(&format!("did:plc:param{i}")), + EventTypeTag::COMMIT, + format!("param-{i}").into_bytes(), + )?; + if i % 4 == 0 { + let _ = writer.sync(); + let _ = faulty_mgr.io().sync_dir(Path::new("/segments")); + } + Ok(()) + })?; + let _ = writer.sync(); + Ok(()) + })(); + + faulty_mgr.shutdown(); + faulty_mgr.io().crash(); + + let faulty_clone = Arc::clone(&faulty_mgr); + let recovery = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> std::io::Result>> { + let recovered_writer = + EventLogWriter::open(Arc::clone(&faulty_clone), 256)?; + + let recovered_seq = recovered_writer.synced_seq().raw(); + assert!( + recovered_seq <= event_count, + "config={config_name} seed={seed}: recovered {recovered_seq} > written {event_count}" + ); + + if recovered_seq == 0 { + return Ok(None); + } + + let fd = faulty_clone.open_for_read(SegmentId::new(1))?; + let events = SegmentReader::open(faulty_clone.io(), fd)? + .valid_prefix()?; + Ok(Some(events)) + })); + + if let Ok(Ok(Some(recovered_events))) = recovery { + let is_prefix = recovered_events + .iter() + .zip(pristine_events.iter()) + .all(|(r, p)| r.seq == p.seq && r.payload == p.payload); + + assert!( + is_prefix, + "config={config_name} seed={seed}: recovered is not prefix of pristine" + ); + } + }); + }); +} diff --git a/crates/tranquil-store/tests/eventlog_properties.rs b/crates/tranquil-store/tests/eventlog_properties.rs new file mode 100644 index 0000000..2701956 --- /dev/null +++ b/crates/tranquil-store/tests/eventlog_properties.rs @@ -0,0 +1,677 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use tranquil_store::eventlog::{ + DidHash, EVENT_RECORD_OVERHEAD, EventLog, EventLogConfig, EventLogReader, EventLogWriter, + EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, PayloadError, RawEvent, SEGMENT_HEADER_SIZE, + SegmentId, SegmentIndex, SegmentManager, SegmentReader, TimestampMicros, ValidEvent, + decode_payload, encode_payload, to_sequenced_event, validate_payload_size, +}; +use tranquil_store::{OpRecord, OpenOptions, SimulatedIO, StorageIO}; + +fn setup_manager(max_segment_size: u64) -> Arc> { + let sim = SimulatedIO::pristine(42); + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()) +} + +fn append_test_event(writer: &mut EventLogWriter, seq_hint: u64) -> EventSequence { + writer + .append( + DidHash::from_did(&format!("did:plc:prop{seq_hint}")), + EventTypeTag::COMMIT, + format!("payload-{seq_hint}").into_bytes(), + ) + .unwrap() +} + +#[test] +fn sequence_assignment_is_contiguous() { + let n = 100u64; + let mgr = setup_manager(64 * 1024); + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + + let seqs: Vec = (1..=n).map(|i| append_test_event(&mut writer, i)).collect(); + + seqs.iter().enumerate().for_each(|(i, seq)| { + assert_eq!( + seq.raw(), + i as u64 + 1, + "event {i} should have seq {}", + i + 1, + ); + }); +} + +#[test] +fn cursor_resumption_returns_correct_suffix() { + let mgr = setup_manager(64 * 1024); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=1000).for_each(|i| { + append_test_event(&mut writer, i); + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + + let events = reader + .read_events_from(EventSequence::new(500), 1000) + .unwrap(); + assert_eq!(events.len(), 500); + assert_eq!(events[0].seq, EventSequence::new(501)); + assert_eq!(events[499].seq, EventSequence::new(1000)); + + events.windows(2).for_each(|pair| { + assert_eq!( + pair[1].seq.raw(), + pair[0].seq.raw() + 1, + "gap between {} and {}", + pair[0].seq, + pair[1].seq, + ); + }); +} + +#[test] +fn cross_segment_read_is_seamless() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let events_per_segment = 10; + let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64; + let total_events = 100u64; + + let mgr = setup_manager(max_segment_size); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=total_events).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:xseg{i}")), + EventTypeTag::COMMIT, + vec![i as u8; payload_size], + ) + .unwrap(); + + if i % events_per_segment as u64 == 0 && i < total_events { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + + let events = reader + .read_events_from(EventSequence::BEFORE_ALL, total_events as usize + 10) + .unwrap(); + + assert_eq!(events.len(), total_events as usize); + + events.iter().enumerate().for_each(|(i, e)| { + assert_eq!( + e.seq, + EventSequence::new(i as u64 + 1), + "event at index {i} has wrong seq" + ); + }); + + let mut seen = std::collections::HashSet::new(); + events.iter().for_each(|e| { + assert!(seen.insert(e.seq.raw()), "duplicate seq {}", e.seq,); + }); +} + +#[test] +fn retention_deletes_only_old_segments() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let events_per_segment = 3; + let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64; + + let sim = SimulatedIO::pristine(42); + let mgr = + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()); + + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + + (1..=15).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:ret{i}")), + EventTypeTag::COMMIT, + vec![0xAA; payload_size], + ) + .unwrap(); + + if i % events_per_segment as u64 == 0 { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + writer.sync().unwrap(); + + let segments_before = mgr.list_segments().unwrap(); + assert!(segments_before.len() >= 5); + + let segments_to_delete: Vec<_> = segments_before[..2].to_vec(); + segments_to_delete.iter().for_each(|&id| { + mgr.delete_segment(id).unwrap(); + }); + + let segments_after = mgr.list_segments().unwrap(); + assert_eq!(segments_after.len(), segments_before.len() - 2,); + + segments_to_delete.iter().for_each(|id| { + assert!( + !segments_after.contains(id), + "deleted segment {id} still present" + ); + }); + + segments_after.iter().for_each(|id| { + assert!( + !segments_to_delete.contains(id), + "remaining segment {id} was supposed to be deleted" + ); + }); +} + +#[test] +fn did_hash_is_deterministic() { + let dids = [ + "did:plc:abc123", + "did:plc:xyz789", + "did:web:example.com", + "did:plc:aaaabbbbccccddddeeeeffffggg", + ]; + + dids.iter().for_each(|did| { + let h1 = DidHash::from_did(did); + let h2 = DidHash::from_did(did); + assert_eq!(h1, h2, "DidHash not deterministic for {did}"); + }); +} + +#[test] +fn payload_round_trip() { + use bytes::Bytes; + use tranquil_db_traits::{AccountStatus, RepoEventType, SequenceNumber, SequencedEvent}; + use tranquil_types::{Did, Handle}; + + let variants: Vec<(RepoEventType, EventTypeTag, SequencedEvent)> = vec![ + ( + RepoEventType::Commit, + EventTypeTag::COMMIT, + SequencedEvent { + seq: SequenceNumber::from_raw(1), + did: Did::new("did:plc:testuser1234567890abcdef").unwrap(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Commit, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: Some( + serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}]), + ), + blobs: Some(vec!["bafkreibtest".to_owned()]), + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: Some("rev1".to_owned()), + }, + ), + ( + RepoEventType::Identity, + EventTypeTag::IDENTITY, + SequencedEvent { + seq: SequenceNumber::from_raw(2), + did: Did::new("did:plc:testuser1234567890abcdef").unwrap(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Identity, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: Some(Handle::new("test.bsky.social").unwrap()), + active: None, + status: None, + rev: None, + }, + ), + ( + RepoEventType::Account, + EventTypeTag::ACCOUNT, + SequencedEvent { + seq: SequenceNumber::from_raw(3), + did: Did::new("did:plc:testuser1234567890abcdef").unwrap(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Account, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: Some(true), + status: Some(AccountStatus::Active), + rev: None, + }, + ), + ( + RepoEventType::Sync, + EventTypeTag::SYNC, + SequencedEvent { + seq: SequenceNumber::from_raw(4), + did: Did::new("did:plc:testuser1234567890abcdef").unwrap(), + created_at: chrono::Utc::now(), + event_type: RepoEventType::Sync, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: None, + }, + ), + ]; + + variants.iter().for_each(|(event_type, tag, event)| { + let encoded = encode_payload(event); + let decoded = decode_payload(&encoded).unwrap(); + + let raw = RawEvent { + seq: EventSequence::new(event.seq.as_i64() as u64), + timestamp: TimestampMicros::now(), + did_hash: DidHash::from_did(event.did.as_str()), + event_type: *tag, + payload: Bytes::from(encoded), + }; + + let reconstructed = to_sequenced_event(&raw, &decoded).unwrap(); + assert_eq!(reconstructed.did.as_str(), event.did.as_str()); + assert_eq!(reconstructed.event_type, *event_type); + assert_eq!(reconstructed.rev, event.rev); + assert_eq!(reconstructed.blobs, event.blobs); + assert_eq!(reconstructed.active, event.active); + }); +} + +#[test] +fn max_payload_accepted() { + let payload = vec![0xBB; MAX_EVENT_PAYLOAD as usize]; + assert!(validate_payload_size(&payload).is_ok()); + + let sim = SimulatedIO::pristine(42); + let dir = Path::new("/test"); + sim.mkdir(dir).unwrap(); + sim.sync_dir(dir).unwrap(); + + let fd = sim + .open(Path::new("/test/segment.tqe"), OpenOptions::read_write()) + .unwrap(); + let mut writer = tranquil_store::eventlog::SegmentWriter::new( + &sim, + fd, + SegmentId::new(1), + EventSequence::new(1), + ) + .unwrap(); + + let event = ValidEvent { + seq: EventSequence::new(1), + timestamp: TimestampMicros::new(1_000_000), + did_hash: DidHash::from_did("did:plc:maxpayload"), + event_type: EventTypeTag::COMMIT, + payload: payload.clone(), + }; + writer.append_event(&sim, &event).unwrap(); + writer.sync(&sim).unwrap(); + + let reader = SegmentReader::open(&sim, fd).unwrap(); + let events = reader.valid_prefix().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].payload.len(), MAX_EVENT_PAYLOAD as usize); +} + +#[test] +fn oversized_payload_rejected() { + let payload = vec![0xCC; MAX_EVENT_PAYLOAD as usize + 1]; + match validate_payload_size(&payload) { + Err(PayloadError::TooLarge { size, max }) => { + assert_eq!(size, MAX_EVENT_PAYLOAD as usize + 1); + assert_eq!(max, MAX_EVENT_PAYLOAD as usize); + } + other => panic!("expected TooLarge, got {other:?}"), + } +} + +#[test] +fn retention_does_not_break_active_readers() { + let payload_size = 50; + let record_size = EVENT_RECORD_OVERHEAD + payload_size; + let events_per_segment = 5; + let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64; + + let sim = SimulatedIO::pristine(42); + let mgr = + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=25).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:active{i}")), + EventTypeTag::COMMIT, + vec![i as u8; payload_size], + ) + .unwrap(); + if i % events_per_segment as u64 == 0 { + writer.sync().unwrap(); + writer.rotate_if_needed().unwrap(); + } + }); + writer.sync().unwrap(); + } + mgr.shutdown(); + + let reader = EventLogReader::new(Arc::clone(&mgr), false); + reader.refresh_segment_ranges().unwrap(); + + let first_batch = reader + .read_events_from(EventSequence::BEFORE_ALL, 10) + .unwrap(); + assert_eq!(first_batch.len(), 10); + + mgr.delete_segment(SegmentId::new(1)).unwrap(); + reader.invalidate_index(SegmentId::new(1)); + reader.invalidate_mmap(SegmentId::new(1)); + reader.refresh_segment_ranges().unwrap(); + + let later_events = reader.read_events_from(EventSequence::new(10), 20).unwrap(); + assert!(!later_events.is_empty()); + later_events.iter().for_each(|e| { + assert!(e.seq.raw() > 10); + }); +} + +#[tokio::test] +async fn subscriber_lag_recovery() { + let sim = SimulatedIO::pristine(42); + let config = EventLogConfig { + segments_dir: PathBuf::from("/segments"), + max_segment_size: 64 * 1024, + index_interval: 256, + broadcast_buffer: 4, + use_mmap: false, + }; + + let event_log = EventLog::open(config, sim).unwrap(); + let mut subscriber = event_log.subscriber(EventSequence::BEFORE_ALL); + + let total_events = 20u64; + (1..=total_events).for_each(|i| { + event_log + .append_and_sync( + &tranquil_types::Did::new("did:plc:testuser1234567890abcdef").unwrap(), + tranquil_db_traits::RepoEventType::Commit, + &tranquil_db_traits::SequencedEvent { + seq: tranquil_db_traits::SequenceNumber::from_raw(i as i64), + did: tranquil_types::Did::new("did:plc:testuser1234567890abcdef").unwrap(), + created_at: chrono::Utc::now(), + event_type: tranquil_db_traits::RepoEventType::Commit, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: None, + }, + ) + .unwrap(); + }); + + let mut received_seqs: Vec = Vec::new(); + let timeout = tokio::time::timeout(Duration::from_secs(5), async { + while let Some(event) = subscriber.next().await { + received_seqs.push(event.seq.raw()); + if event.seq.raw() >= total_events { + break; + } + } + }); + + timeout + .await + .expect("subscriber timed out before receiving all events"); + + assert_eq!( + received_seqs.len(), + total_events as usize, + "subscriber should receive all {total_events} events, got {}", + received_seqs.len(), + ); + + received_seqs.windows(2).for_each(|pair| { + assert!( + pair[1] > pair[0], + "events must be in order: {} -> {}", + pair[0], + pair[1], + ); + }); + + let unique: std::collections::HashSet = received_seqs.iter().copied().collect(); + assert_eq!( + unique.len(), + received_seqs.len(), + "no duplicate events allowed" + ); +} + +#[test] +fn index_checkpoint_accelerates_recovery() { + let event_count = 50_000u64; + let sim = SimulatedIO::pristine(42); + let mgr = + Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), 256 * 1024 * 1024).unwrap()); + + { + let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap(); + (1..=event_count).for_each(|i| { + writer + .append( + DidHash::from_did(&format!("did:plc:chk{i}")), + EventTypeTag::COMMIT, + format!("ckpt-{i}").into_bytes(), + ) + .unwrap(); + }); + writer.shutdown().unwrap(); + } + mgr.shutdown(); + + let index = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1))) + .unwrap() + .unwrap(); + + assert!(index.entry_count() > 0); + assert_eq!(index.first_seq(), Some(EventSequence::new(1))); + assert_eq!(index.last_seq(), Some(EventSequence::new(event_count))); + + let mid = EventSequence::new(event_count / 2); + let offset = index.lookup(mid); + assert!(offset.is_some(), "index should cover midpoint seq {}", mid,); + + let reader_with_index = EventLogReader::new(Arc::clone(&mgr), false); + + let reads_before = mgr + .io() + .op_log() + .iter() + .filter(|op| matches!(op, OpRecord::ReadAt { .. })) + .count(); + + reader_with_index.refresh_segment_ranges().unwrap(); + let mid_events = reader_with_index + .read_events_from(EventSequence::new(event_count / 2), 10) + .unwrap(); + assert_eq!(mid_events.len(), 10); + + let reads_with_index = mgr + .io() + .op_log() + .iter() + .filter(|op| matches!(op, OpRecord::ReadAt { .. })) + .count() + - reads_before; + + let _ = mgr.io().delete(&mgr.index_path(SegmentId::new(1))); + + let reader_without_index = EventLogReader::new(Arc::clone(&mgr), false); + + let reads_before = mgr + .io() + .op_log() + .iter() + .filter(|op| matches!(op, OpRecord::ReadAt { .. })) + .count(); + + reader_without_index.refresh_segment_ranges().unwrap(); + let mid_events_no_idx = reader_without_index + .read_events_from(EventSequence::new(event_count / 2), 10) + .unwrap(); + assert_eq!(mid_events_no_idx.len(), 10); + + let reads_without_index = mgr + .io() + .op_log() + .iter() + .filter(|op| matches!(op, OpRecord::ReadAt { .. })) + .count() + - reads_before; + + assert!( + reads_with_index < reads_without_index, + "read with index ({reads_with_index} reads) should require fewer reads than without ({reads_without_index} reads)" + ); +} + +#[test] +fn fsync_ordering_blocks_before_events() { + use tranquil_store::blockstore::{ + CID_SIZE, DataFileId, DataFileManager, DataFileReader, DataFileWriter, + }; + + fn test_cid(seed: u8) -> [u8; CID_SIZE] { + let mut cid = [0u8; CID_SIZE]; + cid[0] = 0x01; + cid[1] = 0x71; + cid[2] = 0x12; + cid[3] = 0x20; + cid[4] = seed; + cid + } + + let sim = Arc::new(SimulatedIO::pristine(42)); + let data_dir = Path::new("/blocks"); + sim.mkdir(data_dir).unwrap(); + sim.sync_dir(data_dir).unwrap(); + let seg_dir = Path::new("/segments"); + + let block_mgr = + DataFileManager::with_default_max_size(Arc::clone(&sim), data_dir.to_path_buf()); + let event_mgr = Arc::new( + SegmentManager::new(Arc::clone(&sim), PathBuf::from("/segments"), 64 * 1024).unwrap(), + ); + + let block_fd = block_mgr.open_for_append(DataFileId::new(0)).unwrap(); + let mut block_writer = + DataFileWriter::new(block_mgr.io(), block_fd, DataFileId::new(0)).unwrap(); + let cid = test_cid(1); + let _ = block_writer.append_block(&cid, &[0xAA; 128]).unwrap(); + block_writer.sync().unwrap(); + sim.sync_dir(data_dir).unwrap(); + + { + let mut event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap(); + event_writer + .append( + DidHash::from_did("did:plc:fsyncorder"), + EventTypeTag::COMMIT, + b"event-before-sync".to_vec(), + ) + .unwrap(); + } + + sim.crash(); + event_mgr.shutdown(); + + let block_fd = sim + .open( + Path::new("/blocks/000000.tqb"), + OpenOptions::read_only_existing(), + ) + .unwrap(); + let block_reader = DataFileReader::open(&*sim, block_fd).unwrap(); + let recovered_blocks = block_reader.valid_blocks().unwrap(); + assert_eq!( + recovered_blocks.len(), + 1, + "blockstore was synced, block must survive crash" + ); + assert_eq!(recovered_blocks[0].1, cid, "recovered block CID must match"); + + let event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap(); + assert_eq!( + event_writer.synced_seq(), + EventSequence::BEFORE_ALL, + "crash between blockstore sync and eventlog sync must not persist the event (blocks exist, event does not = orphan, not inconsistency)" + ); + + drop(event_writer); + + { + let mut event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap(); + event_writer + .append( + DidHash::from_did("did:plc:fsyncorder"), + EventTypeTag::COMMIT, + b"event-with-sync".to_vec(), + ) + .unwrap(); + event_writer.sync().unwrap(); + sim.sync_dir(seg_dir).unwrap(); + } + + event_mgr.shutdown(); + sim.crash(); + + let event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap(); + assert_eq!( + event_writer.synced_seq(), + EventSequence::new(1), + "both stores synced, event must survive crash" + ); +}