feat(tranquil-store): eventlog

Lewis: May this revision serve well! <lu5a@proton.me>
This commit is contained in:
Lewis
2026-04-10 13:54:41 +03:00
parent a9e3ec63e1
commit 9ea33def13
20 changed files with 8423 additions and 10 deletions
Generated
+14
View File
@@ -4210,6 +4210,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "memmap2"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
dependencies = [
"libc",
]
[[package]]
name = "metrics"
version = "0.24.3"
@@ -7822,6 +7831,7 @@ version = "0.4.7"
dependencies = [
"async-trait",
"bytes",
"chrono",
"cid",
"fjall",
"flume 0.11.1",
@@ -7837,12 +7847,16 @@ dependencies = [
"rand 0.8.5",
"serde",
"serde_ipld_dagcbor",
"serde_json",
"sha2",
"sqlx",
"tempfile",
"thiserror 2.0.18",
"tokio",
"tracing",
"tranquil-db-traits",
"tranquil-repo",
"tranquil-types",
"xxhash-rust",
]
+21
View File
@@ -17,6 +17,11 @@ tokio = { workspace = true, features = ["sync", "rt"] }
bytes = "1"
memmap2 = "0.9"
tracing = { workspace = true }
chrono = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tranquil-db-traits = { workspace = true }
tranquil-types = { workspace = true }
jacquard-repo = { workspace = true }
cid = { workspace = true }
multihash = { workspace = true }
@@ -42,3 +47,19 @@ tikv-jemallocator = "0.6"
[[bench]]
name = "blockstore"
harness = false
[[bench]]
name = "eventlog"
harness = false
[[bench]]
name = "metastore"
harness = false
[[bench]]
name = "metastore_scale"
harness = false
[[bench]]
name = "profile_reads"
harness = false
+959
View File
@@ -0,0 +1,959 @@
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use chrono::Utc;
use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent};
use tranquil_types::Did;
use tranquil_store::RealIO;
use tranquil_store::eventlog::{EventLog, EventLogConfig, EventSequence};
fn make_did(index: usize) -> Did {
let suffix: String = format!("{index:024x}");
Did::new(format!("did:plc:{suffix}")).unwrap()
}
fn make_event(index: usize) -> SequencedEvent {
let ops_size = match index % 4 {
0 => 64,
1 => 256,
2 => 1024,
_ => 4096,
};
let ops_payload: String = (0..ops_size)
.map(|i| ((index.wrapping_mul(31).wrapping_add(i)) % 26 + 97) as u8 as char)
.collect();
SequencedEvent {
seq: SequenceNumber::from_raw(i64::try_from(index + 1).expect("event index overflow")),
did: make_did(index % 10_000),
created_at: Utc::now(),
event_type: match index % 4 {
0 => RepoEventType::Commit,
1 => RepoEventType::Identity,
2 => RepoEventType::Account,
_ => RepoEventType::Sync,
},
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: Some(serde_json::json!({ "data": ops_payload })),
blobs: None,
blocks_cids: None,
handle: None,
active: None,
status: None,
rev: None,
}
}
fn estimated_payload_size(event: &SequencedEvent) -> usize {
tranquil_store::eventlog::encode_payload(event).len()
}
struct LatencyStats {
p50: Duration,
p95: Duration,
p99: Duration,
max: Duration,
mean: Duration,
}
fn compute_stats(durations: &mut [Duration]) -> Option<LatencyStats> {
if durations.is_empty() {
return None;
}
durations.sort();
let len = durations.len();
let sum: Duration = durations.iter().sum();
let divisor = u32::try_from(len).unwrap_or(u32::MAX);
let last = len - 1;
Some(LatencyStats {
p50: durations[last * 50 / 100],
p95: durations[last * 95 / 100],
p99: durations[last * 99 / 100],
max: durations[last],
mean: sum / divisor,
})
}
fn format_latency(stats: Option<&LatencyStats>) -> String {
match stats {
Some(s) => format!(
" | p50={:?} p95={:?} p99={:?} max={:?} mean={:?}",
s.p50, s.p95, s.p99, s.max, s.mean
),
None => String::new(),
}
}
fn open_eventlog(dir: &Path) -> EventLog<RealIO> {
let segments_dir = dir.join("segments");
std::fs::create_dir_all(&segments_dir).unwrap();
EventLog::open(
EventLogConfig {
segments_dir,
..EventLogConfig::default()
},
RealIO::new(),
)
.unwrap()
}
fn bench_sequential_append(event_count: usize) {
println!("-- sequential append: {event_count} events --");
let dir = tempfile::TempDir::new().unwrap();
let log = open_eventlog(dir.path());
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
let total_bytes: usize = events.iter().map(estimated_payload_size).sum();
let mut latencies = Vec::with_capacity(event_count);
let start = Instant::now();
events.iter().enumerate().for_each(|(i, event)| {
let t = Instant::now();
log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
.unwrap();
if (i + 1) % 256 == 0 {
log.sync().unwrap();
}
latencies.push(t.elapsed());
});
log.sync().unwrap();
let elapsed = start.elapsed();
let lat = format_latency(compute_stats(&mut latencies).as_ref());
println!(
"{:.0} events/sec, {:.1} MB/sec, {:.1}ms{lat}",
event_count as f64 / elapsed.as_secs_f64(),
total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0),
elapsed.as_secs_f64() * 1000.0,
);
let _ = log.shutdown();
}
fn bench_concurrent_producers(event_count: usize, producers: usize) {
println!("-- {producers} concurrent producers, {event_count} total events --");
let dir = tempfile::TempDir::new().unwrap();
let log = Arc::new(open_eventlog(dir.path()));
let events_per_producer = event_count / producers;
let actual_count = events_per_producer * producers;
let avg_payload: usize = (0..4)
.map(|i| estimated_payload_size(&make_event(i)))
.sum::<usize>()
/ 4;
let start = Instant::now();
let handles: Vec<_> = (0..producers)
.map(|pid| {
let log = Arc::clone(&log);
std::thread::spawn(move || {
let mut latencies = Vec::with_capacity(events_per_producer);
(0..events_per_producer).for_each(|i| {
let global = pid * events_per_producer + i;
let event = make_event(global);
let t = Instant::now();
log.append_and_sync(&make_did(global % 10_000), RepoEventType::Commit, &event)
.unwrap();
latencies.push(t.elapsed());
});
latencies
})
})
.collect();
let mut all_latencies: Vec<Duration> = handles
.into_iter()
.flat_map(|h| h.join().unwrap())
.collect();
let elapsed = start.elapsed();
let total_bytes = actual_count * avg_payload;
let lat = format_latency(compute_stats(&mut all_latencies).as_ref());
println!(
"{:.0} events/sec, {:.1} MB/sec, {:.1}ms{lat}",
actual_count as f64 / elapsed.as_secs_f64(),
total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0),
elapsed.as_secs_f64() * 1000.0,
);
let _ = log.shutdown();
}
fn bench_batch_append(event_count: usize, batch_size: usize) {
println!("-- batch append: {event_count} events, batch_size={batch_size} --");
let dir = tempfile::TempDir::new().unwrap();
let log = open_eventlog(dir.path());
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
let dids: Vec<Did> = (0..event_count).map(|i| make_did(i % 10_000)).collect();
let total_bytes: usize = events.iter().map(estimated_payload_size).sum();
let mut batch_latencies = Vec::with_capacity(event_count / batch_size + 1);
let start = Instant::now();
events
.chunks(batch_size)
.enumerate()
.for_each(|(chunk_idx, chunk)| {
let base = chunk_idx * batch_size;
let batch: Vec<(&Did, RepoEventType, &SequencedEvent)> = chunk
.iter()
.enumerate()
.map(|(j, event)| (&dids[base + j], RepoEventType::Commit, event))
.collect();
let t = Instant::now();
log.append_batch(batch).unwrap();
log.sync().unwrap();
batch_latencies.push(t.elapsed());
});
let elapsed = start.elapsed();
let lat = format_latency(compute_stats(&mut batch_latencies).as_ref());
println!(
"{:.0} events/sec, {:.1} MB/sec, {:.1}ms{lat}",
event_count as f64 / elapsed.as_secs_f64(),
total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0),
elapsed.as_secs_f64() * 1000.0,
);
let _ = log.shutdown();
}
fn bench_rotation_under_load(event_count: usize) {
println!("-- rotation: {event_count} events, 256KB segments --");
let dir = tempfile::TempDir::new().unwrap();
let segments_dir = dir.path().join("segments");
std::fs::create_dir_all(&segments_dir).unwrap();
let log = EventLog::open(
EventLogConfig {
segments_dir,
max_segment_size: 256 * 1024,
..EventLogConfig::default()
},
RealIO::new(),
)
.unwrap();
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
let append_latencies = Vec::with_capacity(event_count);
let rotation_latencies = Vec::new();
let start = Instant::now();
let (mut append_latencies, mut rotation_latencies, _) = events.iter().enumerate().fold(
(append_latencies, rotation_latencies, false),
|(mut appends, mut rotations, fd_limited), (i, event)| {
let t = Instant::now();
log.append_and_sync(&make_did(i % 10_000), RepoEventType::Commit, event)
.unwrap();
appends.push(t.elapsed());
match fd_limited {
true => (appends, rotations, true),
false => {
let rt = Instant::now();
match log.maybe_rotate() {
Ok(true) => {
rotations.push(rt.elapsed());
(appends, rotations, false)
}
Ok(false) => (appends, rotations, false),
Err(e) => {
println!("fd limit hit at {} segments: {e}", log.segment_count());
(appends, rotations, true)
}
}
}
}
},
);
let elapsed = start.elapsed();
let append_lat = format_latency(compute_stats(&mut append_latencies).as_ref());
let rotation_lat = format_latency(compute_stats(&mut rotation_latencies).as_ref());
println!(
"{:.0} events/sec, {} segments, {} rotations, {:.1}ms",
event_count as f64 / elapsed.as_secs_f64(),
log.segment_count(),
rotation_latencies.len(),
elapsed.as_secs_f64() * 1000.0,
);
println!("append{append_lat}");
println!("rotation{rotation_lat}");
let _ = log.shutdown();
}
fn scan_all_events(log: &EventLog<RealIO>, batch_size: usize) -> usize {
scan_all_events_from(log, EventSequence::BEFORE_ALL, batch_size, 0)
}
fn scan_all_events_from(
log: &EventLog<RealIO>,
cursor: EventSequence,
batch_size: usize,
accumulated: usize,
) -> usize {
let batch = log.get_events_since(cursor, batch_size).unwrap();
match batch.last() {
None => accumulated,
Some(last) => {
let next_cursor = EventSequence::new(u64::try_from(last.seq.as_i64()).unwrap());
scan_all_events_from(log, next_cursor, batch_size, accumulated + batch.len())
}
}
}
fn bench_sequential_scan(event_count: usize) {
println!("-- sequential scan: {event_count} events --");
let dir = tempfile::TempDir::new().unwrap();
let log = open_eventlog(dir.path());
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
let total_bytes: usize = events.iter().map(estimated_payload_size).sum();
events.iter().enumerate().for_each(|(i, event)| {
log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
.unwrap();
});
log.sync().unwrap();
let start = Instant::now();
let read_count = scan_all_events(&log, 4096);
let elapsed = start.elapsed();
println!(
"{:.0} events/sec, {:.1} MB/sec, {read_count} events, {:.1}ms",
read_count as f64 / elapsed.as_secs_f64(),
total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0),
elapsed.as_secs_f64() * 1000.0,
);
let _ = log.shutdown();
}
fn bench_parallel_readers(event_count: usize, readers: usize) {
println!("-- parallel readers: {event_count} events, {readers} readers --");
let dir = tempfile::TempDir::new().unwrap();
let log = Arc::new(open_eventlog(dir.path()));
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
events.iter().enumerate().for_each(|(i, event)| {
log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
.unwrap();
});
log.sync().unwrap();
let total_read = Arc::new(AtomicU64::new(0));
let batch_size = 4096;
let start = Instant::now();
let handles: Vec<_> = (0..readers)
.map(|_| {
let log = Arc::clone(&log);
let total_read = Arc::clone(&total_read);
std::thread::spawn(move || {
let count = scan_all_events(&log, batch_size) as u64;
total_read.fetch_add(count, Ordering::Relaxed);
})
})
.collect();
handles.into_iter().for_each(|h| h.join().unwrap());
let elapsed = start.elapsed();
let total = total_read.load(Ordering::Relaxed);
let avg_payload: usize = (0..4)
.map(|i| estimated_payload_size(&make_event(i)))
.sum::<usize>()
/ 4;
println!(
"{:.0} total events/sec across {readers} readers ({:.0} per reader)",
total as f64 / elapsed.as_secs_f64(),
(total as f64 / readers as f64) / elapsed.as_secs_f64(),
);
println!(
"aggregate {:.1} MB/sec, {:.1}ms",
(total as f64 * avg_payload as f64) / elapsed.as_secs_f64() / (1024.0 * 1024.0),
elapsed.as_secs_f64() * 1000.0,
);
let _ = log.shutdown();
}
fn bench_stampede(event_count: usize, producers: usize, readers: usize, subscribers: usize) {
println!(
"-- stampede: {event_count} events, {producers} producers, {readers} readers, {subscribers} subscribers --"
);
let dir = tempfile::TempDir::new().unwrap();
let log = Arc::new(open_eventlog(dir.path()));
let events_per_producer = event_count / producers;
let actual_events = events_per_producer * producers;
let writes_done = Arc::new(AtomicBool::new(false));
let total_written = Arc::new(AtomicU64::new(0));
let total_read = Arc::new(AtomicU64::new(0));
let total_subscribed = Arc::new(AtomicU64::new(0));
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.enable_all()
.build()
.unwrap();
let start = Instant::now();
let subscriber_handles: Vec<_> = (0..subscribers)
.map(|_| {
let log = Arc::clone(&log);
let writes_done = Arc::clone(&writes_done);
let total_subscribed = Arc::clone(&total_subscribed);
let total_written = Arc::clone(&total_written);
rt.spawn(async move {
let mut sub = log.subscriber(EventSequence::BEFORE_ALL);
let mut count = 0u64;
loop {
match tokio::time::timeout(Duration::from_millis(100), sub.next()).await {
Ok(Some(_)) => {
count += 1;
}
Ok(None) => break,
Err(_) => {
if writes_done.load(Ordering::Acquire) {
let written = total_written.load(Ordering::Acquire);
if count >= written {
break;
}
match tokio::time::timeout(Duration::from_secs(2), sub.next()).await
{
Ok(Some(_)) => count += 1,
_ => break,
}
}
}
}
}
total_subscribed.fetch_add(count, Ordering::Relaxed);
})
})
.collect();
let writer_handles: Vec<_> = (0..producers)
.map(|pid| {
let log = Arc::clone(&log);
let total_written = Arc::clone(&total_written);
std::thread::spawn(move || {
let mut latencies = Vec::with_capacity(events_per_producer);
(0..events_per_producer).for_each(|i| {
let global = pid * events_per_producer + i;
let event = make_event(global);
let t = Instant::now();
log.append_and_sync(&make_did(global % 10_000), RepoEventType::Commit, &event)
.unwrap();
latencies.push(t.elapsed());
total_written.fetch_add(1, Ordering::Release);
});
latencies
})
})
.collect();
let reader_handles: Vec<_> = (0..readers)
.map(|_| {
let log = Arc::clone(&log);
let writes_done = Arc::clone(&writes_done);
let total_read = Arc::clone(&total_read);
std::thread::spawn(move || {
let mut cursor = EventSequence::BEFORE_ALL;
let mut count = 0u64;
loop {
let batch = log.get_events_since(cursor, 1024).unwrap();
match batch.last() {
Some(last) => {
count += batch.len() as u64;
cursor = EventSequence::new(u64::try_from(last.seq.as_i64()).unwrap());
}
None if writes_done.load(Ordering::Acquire) => {
let final_batch = log.get_events_since(cursor, 1024).unwrap();
match final_batch.last() {
Some(last) => {
count += final_batch.len() as u64;
cursor = EventSequence::new(
u64::try_from(last.seq.as_i64()).unwrap(),
);
}
None => break,
}
}
None => {
std::thread::yield_now();
}
}
}
total_read.fetch_add(count, Ordering::Relaxed);
})
})
.collect();
let mut write_latencies: Vec<Duration> = writer_handles
.into_iter()
.flat_map(|h| h.join().unwrap())
.collect();
let write_elapsed = start.elapsed();
writes_done.store(true, Ordering::Release);
reader_handles.into_iter().for_each(|h| h.join().unwrap());
let read_elapsed = start.elapsed();
rt.block_on(async {
let _ = tokio::time::timeout(
Duration::from_secs(10),
futures::future::join_all(subscriber_handles),
)
.await;
});
let total_elapsed = start.elapsed();
let reads = total_read.load(Ordering::Relaxed);
let subscribed = total_subscribed.load(Ordering::Relaxed);
let write_lat = format_latency(compute_stats(&mut write_latencies).as_ref());
println!(
"writes: {:.0} events/sec, {actual_events} events, {:.1}ms{write_lat}",
actual_events as f64 / write_elapsed.as_secs_f64(),
write_elapsed.as_secs_f64() * 1000.0,
);
println!(
"reads: {:.0} events/sec, {readers} readers, {reads} events, {:.1}ms",
reads as f64 / read_elapsed.as_secs_f64(),
read_elapsed.as_secs_f64() * 1000.0,
);
println!(
"subscribers: {subscribed} events across {subscribers} subscribers, {:.1}ms",
total_elapsed.as_secs_f64() * 1000.0,
);
println!("segments: {}", log.segment_count());
let _ = log.shutdown();
}
fn bench_broadcast_fanout(subscriber_count: usize) {
println!("-- broadcast fanout: 10000 events, {subscriber_count} subscribers --");
let dir = tempfile::TempDir::new().unwrap();
let log = Arc::new(open_eventlog(dir.path()));
let event_count = 10_000usize;
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(8),
)
.enable_all()
.build()
.unwrap();
rt.block_on(async {
let received_counts: Arc<Vec<AtomicU64>> =
Arc::new((0..subscriber_count).map(|_| AtomicU64::new(0)).collect());
let sub_handles: Vec<_> = (0..subscriber_count)
.map(|sub_id| {
let mut subscriber = log.subscriber(EventSequence::BEFORE_ALL);
let received_counts = Arc::clone(&received_counts);
tokio::spawn(async move {
let mut count = 0u64;
while count < event_count as u64 {
match tokio::time::timeout(Duration::from_secs(10), subscriber.next()).await
{
Ok(Some(_)) => count += 1,
_ => break,
}
}
received_counts[sub_id].store(count, Ordering::Relaxed);
})
})
.collect();
let log_writer = Arc::clone(&log);
let write_handle = tokio::task::spawn_blocking(move || {
let mut latencies = Vec::with_capacity(event_count);
(0..event_count).for_each(|i| {
let event = make_event(i);
let t = Instant::now();
log_writer
.append_and_sync(&make_did(i % 10_000), RepoEventType::Commit, &event)
.unwrap();
latencies.push(t.elapsed());
});
latencies
});
let mut write_latencies = write_handle.await.unwrap();
let _ = tokio::time::timeout(
Duration::from_secs(30),
futures::future::join_all(sub_handles),
)
.await;
let total_received: u64 = received_counts
.iter()
.map(|c| c.load(Ordering::Relaxed))
.sum();
let min_received = received_counts
.iter()
.map(|c| c.load(Ordering::Relaxed))
.min()
.unwrap_or(0);
let write_lat = format_latency(compute_stats(&mut write_latencies).as_ref());
println!("write{write_lat}");
println!(
"total received: {total_received}/{}, min per sub: {min_received}/{event_count}",
event_count as u64 * subscriber_count as u64,
);
});
let _ = log.shutdown();
}
async fn bench_pg_write_throughput(event_count: usize, concurrency: usize) {
let database_url = match std::env::var("DATABASE_URL") {
Ok(url) => url,
Err(_) => {
println!("skipped, set DATABASE_URL to enable");
return;
}
};
let max_conns = u32::try_from(concurrency)
.unwrap_or(u32::MAX)
.saturating_add(10);
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(max_conns)
.acquire_timeout(Duration::from_secs(30))
.connect(&database_url)
.await
.unwrap();
sqlx::query(
"CREATE TABLE IF NOT EXISTS bench_repo_seq (
seq BIGSERIAL PRIMARY KEY,
did TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
event_type TEXT NOT NULL,
ops JSONB
)",
)
.execute(&pool)
.await
.unwrap();
sqlx::query("TRUNCATE bench_repo_seq")
.execute(&pool)
.await
.unwrap();
let events_per_task = event_count / concurrency;
let actual_count = events_per_task * concurrency;
let start = Instant::now();
let handles: Vec<_> = (0..concurrency)
.map(|task_id| {
let pool = pool.clone();
tokio::spawn(async move {
futures::stream::iter(0..events_per_task)
.then(|i| {
let pool = pool.clone();
async move {
let global = task_id * events_per_task + i;
let did = format!("did:plc:{global:024x}");
let ops_size = match global % 4 {
0 => 64,
1 => 256,
2 => 1024,
_ => 4096,
};
let payload: String = (0..ops_size)
.map(|j| {
((global.wrapping_mul(31).wrapping_add(j)) % 26 + 97) as u8
as char
})
.collect();
let ops = serde_json::json!({ "data": payload });
let t = Instant::now();
sqlx::query(
"INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)",
)
.bind(&did)
.bind("commit")
.bind(&ops)
.execute(&pool)
.await
.unwrap();
t.elapsed()
}
})
.collect::<Vec<Duration>>()
.await
})
})
.collect();
let mut all_latencies: Vec<Duration> = futures::future::join_all(handles)
.await
.into_iter()
.flat_map(Result::unwrap)
.collect();
let elapsed = start.elapsed();
let lat = format_latency(compute_stats(&mut all_latencies).as_ref());
println!(
"{:.0} events/sec, {:.1}ms{lat}",
actual_count as f64 / elapsed.as_secs_f64(),
elapsed.as_secs_f64() * 1000.0,
);
sqlx::query("TRUNCATE bench_repo_seq")
.execute(&pool)
.await
.unwrap();
pool.close().await;
}
async fn bench_pg_read_throughput(event_count: usize, concurrency: usize) {
let database_url = match std::env::var("DATABASE_URL") {
Ok(url) => url,
Err(_) => {
println!("skipped, set DATABASE_URL to enable");
return;
}
};
let max_conns = u32::try_from(concurrency)
.unwrap_or(u32::MAX)
.saturating_add(5);
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(max_conns)
.connect(&database_url)
.await
.unwrap();
sqlx::query(
"CREATE TABLE IF NOT EXISTS bench_repo_seq (
seq BIGSERIAL PRIMARY KEY,
did TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
event_type TEXT NOT NULL,
ops JSONB
)",
)
.execute(&pool)
.await
.unwrap();
let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM bench_repo_seq")
.fetch_one(&pool)
.await
.unwrap();
if (row.0 as usize) < event_count {
sqlx::query("TRUNCATE bench_repo_seq")
.execute(&pool)
.await
.unwrap();
println!("populating {event_count} events");
futures::stream::iter(0..event_count)
.map(|i| {
let pool = pool.clone();
async move {
let did = format!("did:plc:{i:024x}");
let ops = serde_json::json!({ "data": "x".repeat(256) });
sqlx::query(
"INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)",
)
.bind(&did)
.bind("commit")
.bind(&ops)
.execute(&pool)
.await
.unwrap();
}
})
.buffer_unordered(50)
.collect::<Vec<()>>()
.await;
}
let total_events = Arc::new(AtomicU64::new(0));
let start = Instant::now();
let handles: Vec<_> = (0..concurrency)
.map(|_| {
let pool = pool.clone();
let total_events = Arc::clone(&total_events);
tokio::spawn(async move {
let mut cursor = 0i64;
let mut count = 0u64;
loop {
let rows: Vec<(i64,)> = sqlx::query_as(
"SELECT seq FROM bench_repo_seq WHERE seq > $1 ORDER BY seq LIMIT $2",
)
.bind(cursor)
.bind(1000i64)
.fetch_all(&pool)
.await
.unwrap();
if rows.is_empty() {
break;
}
count += rows.len() as u64;
cursor = rows.last().unwrap().0;
}
total_events.fetch_add(count, Ordering::Relaxed);
})
})
.collect();
futures::future::join_all(handles).await;
let elapsed = start.elapsed();
let total = total_events.load(Ordering::Relaxed);
println!(
"{:.0} total events/sec across {concurrency} readers, {total} events, {:.1}ms",
total as f64 / elapsed.as_secs_f64(),
elapsed.as_secs_f64() * 1000.0,
);
pool.close().await;
}
fn main() {
println!("-- eventlog benchmarks --");
let cpus = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(8);
println!("available parallelism: {cpus}");
let parse_env_list = |var: &str, defaults: Vec<usize>| -> Vec<usize> {
std::env::var(var).map_or(defaults, |s| {
s.split(',')
.map(|n| {
n.trim()
.replace('_', "")
.parse::<usize>()
.unwrap_or_else(|e| panic!("{var}: {e}"))
})
.collect()
})
};
let event_counts = parse_env_list("BENCH_EVENT_COUNTS", vec![10_000, 100_000]);
let large_event_counts = parse_env_list("BENCH_LARGE_EVENT_COUNTS", vec![1_000_000]);
let producer_counts = parse_env_list("BENCH_PRODUCERS", vec![1, 10, 50, 100, 500]);
let all_write_counts: Vec<usize> = event_counts
.iter()
.chain(large_event_counts.iter())
.copied()
.collect();
println!("event counts: {event_counts:?}, large: {large_event_counts:?}");
println!("producer counts: {producer_counts:?}");
println!("-- write throughput --");
all_write_counts.iter().for_each(|&n| {
bench_sequential_append(n);
});
all_write_counts.iter().for_each(|&n| {
producer_counts.iter().for_each(|&p| {
if n >= p {
bench_concurrent_producers(n, p);
}
});
});
all_write_counts.iter().for_each(|&n| {
[256usize, 1024, 4096].iter().for_each(|&batch| {
bench_batch_append(n, batch);
});
});
println!("-- rotation --");
event_counts.iter().for_each(|&n| {
bench_rotation_under_load(n);
});
println!("-- read throughput --");
event_counts.iter().for_each(|&n| {
bench_sequential_scan(n);
});
event_counts.iter().for_each(|&n| {
[2usize, 4, 8, 16, 32].iter().for_each(|&r| {
bench_parallel_readers(n, r);
});
});
println!("-- broadcast fanout --");
[1usize, 10, 100, 500, 1000].iter().for_each(|&s| {
bench_broadcast_fanout(s);
});
println!("-- stampede --");
bench_stampede(100_000, 50, 8, 10);
bench_stampede(100_000, 100, 16, 50);
bench_stampede(500_000, 100, 16, 50);
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(cpus)
.enable_all()
.build()
.unwrap();
if std::env::var("DATABASE_URL").is_ok() {
println!("-- postgres comparison --");
event_counts.iter().for_each(|&n| {
producer_counts.iter().for_each(|&p| {
if n >= p {
println!("-- postgres write: {n} events, {p} writers --",);
rt.block_on(bench_pg_write_throughput(n, p));
}
});
});
event_counts.iter().for_each(|&n| {
[1usize, 4, 16, 32].iter().for_each(|&r| {
println!("-- postgres read: {n} events, {r} readers --",);
rt.block_on(bench_pg_read_throughput(n, r));
});
});
rt.block_on(async {
let url = std::env::var("DATABASE_URL").unwrap();
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.unwrap();
sqlx::query("DROP TABLE IF EXISTS bench_repo_seq")
.execute(&pool)
.await
.unwrap();
pool.close().await;
});
} else {
println!("set DATABASE_URL for postgres comparison");
}
}
@@ -4,6 +4,9 @@ use std::io;
use std::sync::Arc;
use std::thread;
use crate::fsync_order::PostBlockstoreHook;
use super::BlocksSynced;
use crate::io::{FileId, OpenOptions, StorageIO};
use super::data_file::{CID_SIZE, DataFileWriter};
@@ -109,6 +112,15 @@ impl GroupCommitWriter {
manager: DataFileManager<S>,
index: Arc<KeyIndex>,
config: GroupCommitConfig,
) -> Result<Self, CommitError> {
Self::spawn_with_hook(manager, index, config, None)
}
pub fn spawn_with_hook<S: StorageIO + 'static>(
manager: DataFileManager<S>,
index: Arc<KeyIndex>,
config: GroupCommitConfig,
post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
) -> Result<Self, CommitError> {
let cursor = index.read_write_cursor().map_err(CommitError::from)?;
let mut state = initialize_active_state(&manager, cursor)?;
@@ -118,7 +130,14 @@ impl GroupCommitWriter {
let handle = thread::Builder::new()
.name("blockstore-group-commit".into())
.spawn(move || {
commit_loop(&manager, &*index, &receiver, &config, &mut state);
commit_loop(
&manager,
&index,
&receiver,
&config,
&mut state,
post_sync_hook.as_deref(),
);
})
.map_err(|e| CommitError::from(io::Error::other(e)))?;
@@ -284,6 +303,7 @@ fn commit_loop<S: StorageIO>(
receiver: &flume::Receiver<CommitRequest>,
config: &GroupCommitConfig,
state: &mut ActiveState,
post_sync_hook: Option<&dyn PostBlockstoreHook>,
) {
loop {
let first = match receiver.recv() {
@@ -302,24 +322,37 @@ fn commit_loop<S: StorageIO>(
let result = process_batch(manager, index, &batch, state);
if let Ok((ref _dedup, ref proof)) = result {
run_post_sync_hook(post_sync_hook, proof);
}
if let Err(ref e) = result {
tracing::warn!(error = %e, "commit batch failed");
}
dispatch_responses(batch, result);
dispatch_responses(batch, result.map(|(dedup, _proof)| dedup));
if shutdown_after {
drain_and_process_remaining(manager, index, receiver, state);
drain_and_process_remaining(manager, index, receiver, state, post_sync_hook);
return;
}
}
}
fn run_post_sync_hook(hook: Option<&dyn PostBlockstoreHook>, proof: &BlocksSynced) {
if let Some(hook) = hook
&& let Err(e) = hook.on_blocks_synced(proof)
{
tracing::error!(error = %e, "post-blockstore sync hook failed");
}
}
fn drain_and_process_remaining<S: StorageIO>(
manager: &DataFileManager<S>,
index: &KeyIndex,
receiver: &flume::Receiver<CommitRequest>,
state: &mut ActiveState,
post_sync_hook: Option<&dyn PostBlockstoreHook>,
) {
let entries: Vec<BatchEntry> = std::iter::from_fn(|| receiver.try_recv().ok())
.filter_map(|req| classify_request(req).ok())
@@ -330,7 +363,12 @@ fn drain_and_process_remaining<S: StorageIO>(
}
let result = process_batch(manager, index, &entries, state);
dispatch_responses(entries, result);
if let Ok((ref _dedup, ref proof)) = result {
run_post_sync_hook(post_sync_hook, proof);
}
dispatch_responses(entries, result.map(|(dedup, _proof)| dedup));
}
struct RotationState {
@@ -343,7 +381,7 @@ fn process_batch<S: StorageIO>(
index: &KeyIndex,
batch: &[BatchEntry],
state: &mut ActiveState,
) -> Result<HashMap<[u8; CID_SIZE], BlockLocation>, CommitError> {
) -> Result<(HashMap<[u8; CID_SIZE], BlockLocation>, BlocksSynced), CommitError> {
let mut dedup: HashMap<[u8; CID_SIZE], BlockLocation> = HashMap::new();
let mut index_entries: Vec<([u8; CID_SIZE], BlockLocation)> = Vec::new();
let mut all_decrements: Vec<[u8; CID_SIZE]> = Vec::new();
@@ -446,7 +484,7 @@ fn process_batch<S: StorageIO>(
.batch_put(&index_entries, &all_decrements, cursor)
.map_err(CommitError::from)?;
Ok(dedup)
Ok((dedup, BlocksSynced::new()))
}
fn dispatch_responses(
@@ -32,6 +32,14 @@ use std::path::Path;
use crate::io::StorageIO;
pub struct BlocksSynced(());
impl BlocksSynced {
pub(in crate::blockstore) fn new() -> Self {
Self(())
}
}
pub(crate) fn list_files_by_extension<S: StorageIO>(
io: &S,
dir: &Path,
+15 -3
View File
@@ -10,6 +10,7 @@ use jacquard_repo::storage::BlockStore;
use multihash::Multihash;
use sha2::{Digest, Sha256};
use crate::fsync_order::PostBlockstoreHook;
use crate::io::{OpenOptions, RealIO, StorageIO};
use super::data_file::{BLOCK_RECORD_OVERHEAD, CID_SIZE, ReadBlockRecord};
@@ -101,6 +102,13 @@ impl Drop for WriterHandle {
impl TranquilBlockStore {
pub fn open(config: BlockStoreConfig) -> Result<Self, RepoError> {
Self::open_with_hook(config, None)
}
pub fn open_with_hook(
config: BlockStoreConfig,
post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
) -> Result<Self, RepoError> {
if config.data_dir == config.index_dir {
return Err(RepoError::storage(io::Error::new(
io::ErrorKind::InvalidInput,
@@ -126,9 +134,13 @@ impl TranquilBlockStore {
let manager_for_writer =
DataFileManager::new(RealIO::new(), config.data_dir.clone(), config.max_file_size);
let writer =
GroupCommitWriter::spawn(manager_for_writer, Arc::clone(&index), config.group_commit)
.map_err(commit_error_to_repo)?;
let writer = GroupCommitWriter::spawn_with_hook(
manager_for_writer,
Arc::clone(&index),
config.group_commit,
post_sync_hook,
)
.map_err(commit_error_to_repo)?;
let sender = writer.sender().clone();
let manager_for_reader = Arc::new(DataFileManager::new(
@@ -0,0 +1,296 @@
use std::io;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use tracing::warn;
use tranquil_db_traits::{DbError, SequenceNumber, SequencedEvent};
use super::notifier::EventLogNotifier;
use super::types::{EventSequence, TimestampMicros};
use super::writer::SyncResult;
use super::{EventLog, EventWithMutations, decode_payload, to_sequenced_event};
use crate::io::StorageIO;
pub struct DeferredBroadcast(SyncResult);
fn io_to_db(e: io::Error) -> DbError {
DbError::Query(e.to_string())
}
fn seq_to_event(seq: SequenceNumber) -> EventSequence {
let raw = seq.as_i64();
if raw < 0 {
warn!(
seq = raw,
"negative SequenceNumber passed to eventlog bridge, treating as BEFORE_ALL"
);
return EventSequence::BEFORE_ALL;
}
EventSequence::cursor_from_i64(raw).unwrap_or(EventSequence::BEFORE_ALL)
}
fn datetime_to_micros(dt: &DateTime<Utc>) -> u64 {
let micros = dt.timestamp_micros();
debug_assert!(micros >= 0, "pre-epoch DateTime passed to eventlog bridge");
u64::try_from(micros).unwrap_or(0)
}
pub struct EventLogBridge<S: StorageIO> {
log: Arc<EventLog<S>>,
}
impl<S: StorageIO> EventLogBridge<S> {
pub fn new(log: Arc<EventLog<S>>) -> Self {
Self { log }
}
pub fn notifier(&self) -> EventLogNotifier<S> {
EventLogNotifier::new(Arc::clone(&self.log))
}
pub fn log(&self) -> &Arc<EventLog<S>> {
&self.log
}
pub fn get_max_seq(&self) -> SequenceNumber {
let es = self.log.max_seq();
SequenceNumber::from_raw(es.as_i64())
}
pub fn get_events_since_seq(
&self,
since: SequenceNumber,
limit: Option<i64>,
) -> Result<Vec<SequencedEvent>, DbError> {
let cap = limit
.and_then(|l| usize::try_from(l).ok())
.unwrap_or(usize::MAX);
self.get_events_impl(since, cap)
}
pub fn get_events_since_cursor(
&self,
cursor: SequenceNumber,
limit: i64,
) -> Result<Vec<SequencedEvent>, DbError> {
let cap = usize::try_from(limit).unwrap_or(usize::MAX);
self.get_events_impl(cursor, cap)
}
fn get_events_impl(
&self,
since: SequenceNumber,
limit: usize,
) -> Result<Vec<SequencedEvent>, DbError> {
let cursor = seq_to_event(since);
self.log.get_events_since(cursor, limit).map_err(io_to_db)
}
pub fn get_event_by_seq(&self, seq: SequenceNumber) -> Result<Option<SequencedEvent>, DbError> {
let es = EventSequence::from_i64(seq.as_i64())
.ok_or_else(|| DbError::Query("invalid sequence number".into()))?;
self.log.get_event(es).map_err(io_to_db)
}
pub fn get_events_in_seq_range(
&self,
start: SequenceNumber,
end: SequenceNumber,
) -> Result<Vec<SequencedEvent>, DbError> {
let end_raw = match u64::try_from(end.as_i64()) {
Ok(v) => v,
Err(_) => return Ok(Vec::new()),
};
let cursor = seq_to_event(start);
if end_raw <= cursor.raw().saturating_add(1) {
return Ok(Vec::new());
}
let range_size =
usize::try_from(end_raw.saturating_sub(cursor.raw())).unwrap_or(usize::MAX);
let raw_events = self
.log
.reader()
.read_events_from(cursor, range_size)
.map_err(io_to_db)?;
raw_events
.iter()
.take_while(|e| e.seq.raw() < end_raw)
.map(|raw| {
let payload =
decode_payload(&raw.payload).map_err(|e| DbError::Query(e.to_string()))?;
to_sequenced_event(raw, &payload).map_err(|e| DbError::Query(e.to_string()))
})
.collect()
}
pub fn get_min_seq_since(
&self,
since: DateTime<Utc>,
) -> Result<Option<SequenceNumber>, DbError> {
let target_us = datetime_to_micros(&since);
let target_ts = TimestampMicros::new(target_us);
let reader = self.log.reader();
let segments = self.log.manager().list_segments().map_err(io_to_db)?;
if segments.is_empty() {
return Ok(None);
}
let scan_from_seg = self.find_segment_for_timestamp(&segments, target_ts)?;
let start_seq = match scan_from_seg {
Some(idx) => reader
.load_index(segments[idx])
.map_err(io_to_db)?
.first_seq()
.map(|s| s.prev_or_before_all())
.unwrap_or(EventSequence::BEFORE_ALL),
None => return Ok(None),
};
const SCAN_BATCH: usize = 1024;
self.scan_for_timestamp(reader, start_seq, target_ts, SCAN_BATCH)
}
fn scan_for_timestamp(
&self,
reader: &super::EventLogReader<S>,
cursor: EventSequence,
target_ts: TimestampMicros,
batch_size: usize,
) -> Result<Option<SequenceNumber>, DbError> {
let batch = reader
.read_events_from(cursor, batch_size)
.map_err(io_to_db)?;
if batch.is_empty() {
return Ok(None);
}
match batch.iter().find(|e| e.timestamp >= target_ts) {
Some(e) => Ok(Some(SequenceNumber::from_raw(e.seq.as_i64()))),
None => {
let next_cursor = batch.last().map(|e| e.seq).unwrap_or(cursor);
self.scan_for_timestamp(reader, next_cursor, target_ts, batch_size)
}
}
}
fn find_segment_for_timestamp(
&self,
segments: &[super::SegmentId],
target_ts: TimestampMicros,
) -> Result<Option<usize>, DbError> {
let reader = self.log.reader();
let last_seg_idx = segments.len() - 1;
let last_index = reader
.load_index(segments[last_seg_idx])
.map_err(io_to_db)?;
let last_ts = last_index
.first_seq()
.and_then(|seq| reader.read_event_at(seq).ok().flatten())
.map(|e| e.timestamp);
match last_ts {
Some(ts) if ts < target_ts => {
let tail_ts = last_index
.last_seq()
.and_then(|seq| reader.read_event_at(seq).ok().flatten())
.map(|e| e.timestamp);
match tail_ts {
Some(ts) if ts < target_ts => return Ok(None),
_ => return Ok(Some(last_seg_idx)),
}
}
None => return Ok(None),
_ => {}
}
Ok(self
.binary_search_segment(reader, segments, target_ts, 0, last_seg_idx, None)?
.map(|r| r.saturating_sub(1)))
}
fn binary_search_segment(
&self,
reader: &super::EventLogReader<S>,
segments: &[super::SegmentId],
target_ts: TimestampMicros,
lo: usize,
hi: usize,
best: Option<usize>,
) -> Result<Option<usize>, DbError> {
if lo > hi {
return Ok(best);
}
let mid = lo + (hi - lo) / 2;
let seg_ts = reader
.load_index(segments[mid])
.map_err(io_to_db)?
.first_seq()
.and_then(|seq| reader.read_event_at(seq).ok().flatten())
.map(|e| e.timestamp);
match seg_ts {
Some(ts) if ts < target_ts => {
self.binary_search_segment(reader, segments, target_ts, mid + 1, hi, best)
}
Some(_) => match mid {
0 => Ok(Some(0)),
_ => {
self.binary_search_segment(reader, segments, target_ts, lo, mid - 1, Some(mid))
}
},
None => self.binary_search_segment(reader, segments, target_ts, mid + 1, hi, best),
}
}
pub fn get_events_with_mutations_since(
&self,
since: SequenceNumber,
limit: usize,
) -> Result<Vec<EventWithMutations>, DbError> {
let cursor = seq_to_event(since);
self.log
.get_events_with_mutations_since(cursor, limit)
.map_err(io_to_db)
}
pub fn insert_event(&self, event: &SequencedEvent) -> Result<SequenceNumber, io::Error> {
let seq = self
.log
.append_and_sync(&event.did, event.event_type, event)?;
Ok(SequenceNumber::from_raw(seq.as_i64()))
}
pub fn insert_event_deferred(
&self,
event: &SequencedEvent,
) -> Result<(SequenceNumber, DeferredBroadcast), io::Error> {
let seq = self.log.append_event(&event.did, event.event_type, event)?;
let sync_result = self.log.sync_data()?;
Ok((
SequenceNumber::from_raw(seq.as_i64()),
DeferredBroadcast(sync_result),
))
}
pub fn insert_event_deferred_raw(
&self,
did: &tranquil_types::Did,
event_type: tranquil_db_traits::RepoEventType,
payload: Vec<u8>,
) -> Result<(SequenceNumber, DeferredBroadcast), io::Error> {
let seq = self.log.append_raw_payload(did, event_type, payload)?;
let sync_result = self.log.sync_data()?;
Ok((
SequenceNumber::from_raw(seq.as_i64()),
DeferredBroadcast(sync_result),
))
}
pub fn complete_broadcast(&self, deferred: DeferredBroadcast) {
self.log.broadcast_result(&deferred.0);
}
}
@@ -0,0 +1,659 @@
use std::collections::HashMap;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use parking_lot::RwLock;
use crate::io::{FileId, OpenOptions, StorageIO};
use super::segment_file::SEGMENT_HEADER_SIZE;
use super::segment_index::SegmentIndex;
use super::types::{SegmentId, SegmentOffset};
pub(crate) const SEGMENT_FILE_EXTENSION: &str = "tqe";
pub(crate) const INDEX_FILE_EXTENSION: &str = "tqi";
struct CachedSegmentHandle {
fd: FileId,
sealed: bool,
writable: bool,
}
pub struct SegmentManager<S: StorageIO> {
io: S,
segments_dir: PathBuf,
max_segment_size: u64,
handles: RwLock<HashMap<SegmentId, CachedSegmentHandle>>,
retention_epoch: AtomicU64,
}
impl<S: StorageIO> SegmentManager<S> {
pub fn new(io: S, segments_dir: PathBuf, max_segment_size: u64) -> io::Result<Self> {
assert!(
max_segment_size > SEGMENT_HEADER_SIZE as u64,
"max_segment_size ({max_segment_size}) must exceed SEGMENT_HEADER_SIZE ({SEGMENT_HEADER_SIZE})"
);
io.mkdir(&segments_dir)?;
Ok(Self {
io,
segments_dir,
max_segment_size,
handles: RwLock::new(HashMap::new()),
retention_epoch: AtomicU64::new(0),
})
}
pub fn io(&self) -> &S {
&self.io
}
pub fn segments_dir(&self) -> &Path {
&self.segments_dir
}
pub fn max_segment_size(&self) -> u64 {
self.max_segment_size
}
pub fn segment_path(&self, id: SegmentId) -> PathBuf {
self.segments_dir
.join(format!("{id}.{SEGMENT_FILE_EXTENSION}"))
}
pub fn index_path(&self, id: SegmentId) -> PathBuf {
self.segments_dir
.join(format!("{id}.{INDEX_FILE_EXTENSION}"))
}
pub fn list_segments(&self) -> io::Result<Vec<SegmentId>> {
let entries = self.io.list_dir(&self.segments_dir)?;
let mut ids: Vec<SegmentId> = entries
.iter()
.filter_map(|path| {
let stem = path.file_stem()?.to_str()?;
let ext = path.extension()?.to_str()?;
(ext == SEGMENT_FILE_EXTENSION)
.then(|| stem.parse::<u32>().ok().map(SegmentId::new))?
})
.collect();
ids.sort();
Ok(ids)
}
pub fn open_for_read(&self, id: SegmentId) -> io::Result<FileId> {
if let Some(entry) = self.handles.read().get(&id) {
return Ok(entry.fd);
}
let path = self.segment_path(id);
let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
let mut cache = self.handles.write();
match cache.get(&id) {
Some(entry) => {
let _ = self.io.close(fd);
Ok(entry.fd)
}
None => {
cache.insert(
id,
CachedSegmentHandle {
fd,
sealed: false,
writable: false,
},
);
Ok(fd)
}
}
}
pub fn open_for_append(&self, id: SegmentId) -> io::Result<FileId> {
{
let cache = self.handles.read();
if let Some(entry) = cache.get(&id) {
if entry.sealed {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("cannot append to sealed segment {id}"),
));
}
if entry.writable {
return Ok(entry.fd);
}
}
}
let path = self.segment_path(id);
let fd = self.io.open(&path, OpenOptions::read_write())?;
let mut cache = self.handles.write();
match cache.get(&id) {
Some(entry) if entry.sealed => {
let _ = self.io.close(fd);
Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("cannot append to sealed segment {id}"),
))
}
Some(entry) if entry.writable => {
let _ = self.io.close(fd);
Ok(entry.fd)
}
Some(entry) => {
let old_fd = entry.fd;
cache.insert(
id,
CachedSegmentHandle {
fd,
sealed: false,
writable: true,
},
);
let _ = self.io.close(old_fd);
Ok(fd)
}
None => {
cache.insert(
id,
CachedSegmentHandle {
fd,
sealed: false,
writable: true,
},
);
Ok(fd)
}
}
}
pub fn should_rotate(&self, position: SegmentOffset) -> bool {
position.raw() >= self.max_segment_size
}
pub fn prepare_rotation(&self, current_id: SegmentId) -> io::Result<(SegmentId, FileId)> {
let next = current_id.next();
let path = self.segment_path(next);
let fd = self.io.open(&path, OpenOptions::read_write())?;
self.io.truncate(fd, 0)?;
self.io.sync_dir(&self.segments_dir)?;
Ok((next, fd))
}
pub fn commit_rotation(&self, new_id: SegmentId, fd: FileId) {
self.handles.write().insert(
new_id,
CachedSegmentHandle {
fd,
sealed: false,
writable: true,
},
);
}
pub fn seal_segment(&self, id: SegmentId, index: &SegmentIndex) -> io::Result<()> {
let path = self.index_path(id);
index.save(&self.io, &path)?;
let mut cache = self.handles.write();
let entry = cache.get_mut(&id).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("seal_segment: segment {id} not in handle cache"),
)
})?;
entry.sealed = true;
Ok(())
}
pub fn is_sealed(&self, id: SegmentId) -> bool {
self.handles
.read()
.get(&id)
.is_some_and(|entry| entry.sealed)
}
pub fn rollback_rotation(&self, new_id: SegmentId, fd: FileId) {
let _ = self.io.close(fd);
self.handles.write().remove(&new_id);
let _ = self.io.delete(&self.segment_path(new_id));
}
pub fn delete_segment(&self, id: SegmentId) -> io::Result<()> {
{
let mut cache = self.handles.write();
if let Some(entry) = cache.remove(&id) {
let _ = self.io.close(entry.fd);
}
}
match self.io.delete(&self.index_path(id)) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::NotFound => {}
Err(e) => return Err(e),
}
self.io.delete(&self.segment_path(id))?;
self.io.sync_dir(&self.segments_dir)?;
self.retention_epoch.fetch_add(1, Ordering::Relaxed);
Ok(())
}
pub fn oldest_segment(&self) -> io::Result<Option<SegmentId>> {
self.list_segments().map(|segs| segs.into_iter().next())
}
pub fn retention_epoch(&self) -> u64 {
self.retention_epoch.load(Ordering::Relaxed)
}
pub fn shutdown(&self) {
self.handles.write().drain().for_each(|(_, handle)| {
let _ = self.io.close(handle.fd);
});
}
}
impl<S: StorageIO> Drop for SegmentManager<S> {
fn drop(&mut self) {
self.shutdown();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::eventlog::segment_file::{SegmentWriter, ValidEvent};
use crate::eventlog::segment_index::{DEFAULT_INDEX_INTERVAL, rebuild_from_segment};
use crate::eventlog::types::{
DidHash, EventSequence, EventTypeTag, SegmentOffset, TimestampMicros,
};
use crate::sim::SimulatedIO;
fn setup_manager(max_segment_size: u64) -> SegmentManager<SimulatedIO> {
let sim = SimulatedIO::pristine(42);
SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()
}
fn test_event(seq: u64, payload: &[u8]) -> ValidEvent {
ValidEvent {
seq: EventSequence::new(seq),
timestamp: TimestampMicros::new(seq * 1_000_000),
did_hash: DidHash::from_did(&format!("did:plc:test{seq}")),
event_type: EventTypeTag::COMMIT,
payload: payload.to_vec(),
}
}
#[test]
fn new_creates_directory() {
let sim = SimulatedIO::pristine(42);
let mgr = SegmentManager::new(sim, PathBuf::from("/eventlog/segments"), 1024).unwrap();
let entries = mgr.io().list_dir(Path::new("/eventlog/segments")).unwrap();
assert!(entries.is_empty());
}
#[test]
fn segment_path_format() {
let mgr = setup_manager(1024);
assert_eq!(
mgr.segment_path(SegmentId::new(0)),
Path::new("/segments/00000000.tqe")
);
assert_eq!(
mgr.segment_path(SegmentId::new(42)),
Path::new("/segments/00000042.tqe")
);
}
#[test]
fn index_path_format() {
let mgr = setup_manager(1024);
assert_eq!(
mgr.index_path(SegmentId::new(0)),
Path::new("/segments/00000000.tqi")
);
assert_eq!(
mgr.index_path(SegmentId::new(7)),
Path::new("/segments/00000007.tqi")
);
}
#[test]
fn open_for_append_creates_file() {
let mgr = setup_manager(1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
}
#[test]
fn open_for_read_missing_file_errors() {
let mgr = setup_manager(1024);
assert!(mgr.open_for_read(SegmentId::new(99)).is_err());
}
#[test]
fn handle_cache_returns_same_fd() {
let mgr = setup_manager(1024);
let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap();
assert_eq!(fd1, fd2);
}
#[test]
fn open_for_read_uses_cache_from_append() {
let mgr = setup_manager(1024);
let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap();
let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
assert_eq!(fd_write, fd_read);
}
#[test]
fn list_segments_finds_segment_files() {
let mgr = setup_manager(1024);
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.open_for_append(SegmentId::new(3)).unwrap();
let segments = mgr.list_segments().unwrap();
assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(3)]);
}
#[test]
fn list_segments_ignores_non_segment_files() {
let mgr = setup_manager(1024);
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.io()
.open(Path::new("/segments/notes.txt"), OpenOptions::read_write())
.unwrap();
let segments = mgr.list_segments().unwrap();
assert_eq!(segments, vec![SegmentId::new(1)]);
}
#[test]
fn list_segments_ignores_index_files() {
let mgr = setup_manager(1024);
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.io()
.open(
Path::new("/segments/00000001.tqi"),
OpenOptions::read_write(),
)
.unwrap();
let segments = mgr.list_segments().unwrap();
assert_eq!(segments, vec![SegmentId::new(1)]);
}
#[test]
fn list_segments_sorted_ascending() {
let mgr = setup_manager(1024);
mgr.open_for_append(SegmentId::new(5)).unwrap();
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.open_for_append(SegmentId::new(3)).unwrap();
let segments = mgr.list_segments().unwrap();
assert_eq!(
segments,
vec![SegmentId::new(1), SegmentId::new(3), SegmentId::new(5)]
);
}
#[test]
fn should_rotate_respects_threshold() {
let mgr = setup_manager(1024);
assert!(!mgr.should_rotate(SegmentOffset::new(100)));
assert!(!mgr.should_rotate(SegmentOffset::new(1023)));
assert!(mgr.should_rotate(SegmentOffset::new(1024)));
assert!(mgr.should_rotate(SegmentOffset::new(2000)));
}
#[test]
fn rotation_lifecycle_prepare_commit() {
let mgr = setup_manager(1024);
let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
assert_eq!(next_id, SegmentId::new(2));
assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
mgr.commit_rotation(next_id, next_fd);
assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
}
#[test]
fn rotation_rollback_cleans_up() {
let mgr = setup_manager(1024);
let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
mgr.commit_rotation(next_id, next_fd);
assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
mgr.rollback_rotation(next_id, next_fd);
let segments = mgr.list_segments().unwrap();
assert_eq!(segments, vec![SegmentId::new(1)]);
}
#[test]
fn seal_segment_persists_index_and_marks_sealed() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer =
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
(1u64..=10).for_each(|i| {
writer
.append_event(mgr.io(), &test_event(i, format!("payload-{i}").as_bytes()))
.unwrap();
});
writer.sync(mgr.io()).unwrap();
let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert!(!mgr.is_sealed(SegmentId::new(1)));
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
assert!(mgr.is_sealed(SegmentId::new(1)));
let loaded = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1)))
.unwrap()
.unwrap();
assert_eq!(loaded, index);
}
#[test]
fn delete_segment_removes_files_and_handle() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer =
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer
.append_event(mgr.io(), &test_event(1, b"will be deleted"))
.unwrap();
writer.sync(mgr.io()).unwrap();
let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap();
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
let epoch_before = mgr.retention_epoch();
mgr.delete_segment(SegmentId::new(1)).unwrap();
assert_eq!(mgr.retention_epoch(), epoch_before + 1);
assert!(mgr.list_segments().unwrap().is_empty());
assert!(mgr.open_for_read(SegmentId::new(1)).is_err());
}
#[test]
fn oldest_segment_returns_first() {
let mgr = setup_manager(1024);
assert_eq!(mgr.oldest_segment().unwrap(), None);
mgr.open_for_append(SegmentId::new(3)).unwrap();
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.open_for_append(SegmentId::new(5)).unwrap();
assert_eq!(mgr.oldest_segment().unwrap(), Some(SegmentId::new(1)));
}
#[test]
fn retention_epoch_starts_at_zero() {
let mgr = setup_manager(1024);
assert_eq!(mgr.retention_epoch(), 0);
}
#[test]
fn rotate_and_write_across_segments() {
let mgr = setup_manager(1024);
let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer1 =
SegmentWriter::new(mgr.io(), fd1, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer1
.append_event(mgr.io(), &test_event(1, b"first segment"))
.unwrap();
writer1.sync(mgr.io()).unwrap();
let (id2, fd2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
mgr.commit_rotation(id2, fd2);
let mut writer2 = SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2)).unwrap();
writer2
.append_event(mgr.io(), &test_event(2, b"second segment"))
.unwrap();
writer2.sync(mgr.io()).unwrap();
let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(events1.len(), 1);
assert_eq!(events1[0].payload, b"first segment");
let fd2_read = mgr.open_for_read(id2).unwrap();
let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(events2.len(), 1);
assert_eq!(events2[0].payload, b"second segment");
}
#[test]
fn seal_then_append_errors() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let index = SegmentIndex::new();
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
let result = mgr.open_for_append(SegmentId::new(1));
assert!(result.is_err());
}
#[test]
fn accessors() {
let mgr = setup_manager(999);
assert_eq!(mgr.max_segment_size(), 999);
assert_eq!(mgr.segments_dir(), Path::new("/segments"));
}
#[test]
fn multiple_deletions_increment_epoch() {
let mgr = setup_manager(1024);
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.open_for_append(SegmentId::new(2)).unwrap();
mgr.open_for_append(SegmentId::new(3)).unwrap();
assert_eq!(mgr.retention_epoch(), 0);
mgr.delete_segment(SegmentId::new(1)).unwrap();
assert_eq!(mgr.retention_epoch(), 1);
mgr.delete_segment(SegmentId::new(2)).unwrap();
assert_eq!(mgr.retention_epoch(), 2);
}
#[test]
fn open_for_read_does_not_infer_sealed_from_index_file() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer =
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer
.append_event(mgr.io(), &test_event(1, b"sealed test"))
.unwrap();
writer.sync(mgr.io()).unwrap();
let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap();
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
mgr.handles.write().remove(&SegmentId::new(1));
let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
assert!(!mgr.is_sealed(SegmentId::new(1)));
}
#[test]
fn open_for_read_unsealed_allows_append() {
let mgr = setup_manager(1024);
let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.handles.write().remove(&SegmentId::new(1));
let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
assert!(!mgr.is_sealed(SegmentId::new(1)));
}
#[test]
fn shutdown_clears_handles() {
let mgr = setup_manager(1024);
mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.open_for_append(SegmentId::new(2)).unwrap();
mgr.shutdown();
assert!(mgr.handles.read().is_empty());
}
#[test]
#[should_panic(expected = "max_segment_size")]
fn rejects_max_segment_size_too_small() {
let sim = SimulatedIO::pristine(42);
let _ = SegmentManager::new(sim, PathBuf::from("/segments"), 5);
}
#[test]
fn prepare_rotation_truncates_stale_file() {
let mgr = setup_manager(1024);
let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
let stale_path = mgr.segment_path(SegmentId::new(2));
let stale_fd = mgr
.io()
.open(&stale_path, OpenOptions::read_write())
.unwrap();
mgr.io().write_all_at(stale_fd, 0, &[0xDE; 4096]).unwrap();
mgr.io().sync(stale_fd).unwrap();
assert_eq!(mgr.io().file_size(stale_fd).unwrap(), 4096);
mgr.io().close(stale_fd).unwrap();
let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
assert_eq!(next_id, SegmentId::new(2));
assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
}
#[test]
fn open_for_append_upgrades_read_only_handle() {
let mgr = setup_manager(1024);
let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap();
mgr.handles.write().remove(&SegmentId::new(1));
let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
assert_ne!(fd_read, fd_append);
assert!(!mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);
let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap();
assert_ne!(fd_upgraded, fd_read);
assert!(mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);
}
#[test]
fn seal_uncached_segment_returns_error() {
let mgr = setup_manager(1024);
let index = SegmentIndex::new();
let result = mgr.seal_segment(SegmentId::new(99), &index);
assert!(result.is_err());
}
}
+489
View File
@@ -0,0 +1,489 @@
mod bridge;
mod manager;
mod notifier;
mod payload;
mod reader;
mod segment_file;
mod segment_index;
mod types;
mod writer;
use std::collections::VecDeque;
use std::io;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use parking_lot::Mutex;
use tokio::sync::broadcast;
use tracing::warn;
use tranquil_db_traits::{RepoEventType, SequencedEvent};
use tranquil_types::Did;
use crate::blockstore::BlocksSynced;
use crate::fsync_order::PostBlockstoreHook;
use crate::io::StorageIO;
pub use bridge::{DeferredBroadcast, EventLogBridge};
pub use manager::SegmentManager;
pub use notifier::EventLogNotifier;
pub use payload::{
EventPayload, PayloadError, decode_payload, encode_payload, encode_payload_with_mutations,
to_sequenced_event, validate_payload_size,
};
pub use reader::{EventLogReader, RawEvent};
pub use segment_file::{
EVENT_HEADER_SIZE, EVENT_RECORD_OVERHEAD, ReadEventRecord, SEGMENT_FORMAT_VERSION,
SEGMENT_HEADER_SIZE, SEGMENT_MAGIC, SegmentReader, SegmentWriter, ValidEvent,
ValidateEventRecord, decode_event_record, encode_event_record, validate_event_record,
};
pub use segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment};
pub use types::{
DEFAULT_SEGMENT_SIZE, DidHash, EventLength, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD,
SegmentId, SegmentOffset, TimestampMicros,
};
pub use writer::{EventLogWriter, SyncResult};
const DEFAULT_BROADCAST_BUFFER: usize = 16384;
pub struct EventWithMutations {
pub event: SequencedEvent,
pub mutation_set: Option<Vec<u8>>,
}
pub struct EventLogConfig {
pub segments_dir: PathBuf,
pub max_segment_size: u64,
pub index_interval: usize,
pub broadcast_buffer: usize,
pub use_mmap: bool,
}
impl Default for EventLogConfig {
fn default() -> Self {
Self {
segments_dir: PathBuf::from("eventlog"),
max_segment_size: DEFAULT_SEGMENT_SIZE,
index_interval: DEFAULT_INDEX_INTERVAL,
broadcast_buffer: DEFAULT_BROADCAST_BUFFER,
use_mmap: true,
}
}
}
pub struct EventLog<S: StorageIO> {
writer: Mutex<EventLogWriter<S>>,
reader: Arc<EventLogReader<S>>,
manager: Arc<SegmentManager<S>>,
broadcast_tx: broadcast::Sender<RawEvent>,
synced_seq: AtomicU64,
consecutive_sync_failures: AtomicU32,
}
impl<S: StorageIO> EventLog<S> {
pub fn open(config: EventLogConfig, io: S) -> io::Result<Self> {
let manager = Arc::new(SegmentManager::new(
io,
config.segments_dir,
config.max_segment_size,
)?);
let writer = EventLogWriter::open(Arc::clone(&manager), config.index_interval)?;
let synced = writer.synced_seq();
let reader = Arc::new(EventLogReader::new(Arc::clone(&manager), config.use_mmap));
reader.set_active_segment(writer.active_segment_id());
reader.seed_index(writer.active_segment_id(), writer.active_index_snapshot());
reader.refresh_segment_ranges()?;
let (broadcast_tx, _) = broadcast::channel(config.broadcast_buffer);
Ok(Self {
writer: Mutex::new(writer),
reader,
manager,
broadcast_tx,
synced_seq: AtomicU64::new(synced.raw()),
consecutive_sync_failures: AtomicU32::new(0),
})
}
pub fn append_event(
&self,
did: &Did,
event_type: RepoEventType,
event: &SequencedEvent,
) -> io::Result<EventSequence> {
let payload = encode_payload(event);
self.append_raw_payload(did, event_type, payload)
}
pub fn append_raw_payload(
&self,
did: &Did,
event_type: RepoEventType,
payload: Vec<u8>,
) -> io::Result<EventSequence> {
let did_hash = DidHash::from_did(did.as_str());
let tag = repo_event_type_to_tag(event_type);
validate_payload_size(&payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
self.writer.lock().append(did_hash, tag, payload)
}
pub fn sync(&self) -> io::Result<SyncResult> {
self.sync_and_broadcast()
}
pub fn append_and_sync(
&self,
did: &Did,
event_type: RepoEventType,
event: &SequencedEvent,
) -> io::Result<EventSequence> {
let seq = self.append_event(did, event_type, event)?;
self.sync_and_broadcast()?;
Ok(seq)
}
pub fn append_batch(
&self,
events: Vec<(&Did, RepoEventType, &SequencedEvent)>,
) -> io::Result<Vec<EventSequence>> {
let mut writer = self.writer.lock();
events
.iter()
.map(|(did, event_type, event)| {
let did_hash = DidHash::from_did(did.as_str());
let tag = repo_event_type_to_tag(*event_type);
let payload = encode_payload(event);
validate_payload_size(&payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
writer.append(did_hash, tag, payload)
})
.collect()
}
pub fn sync_data(&self) -> io::Result<SyncResult> {
let mut writer = self.writer.lock();
let result = writer.sync()?;
self.synced_seq
.store(result.synced_through.raw(), Ordering::Release);
if let (Some(first), Some(last)) =
(result.flushed_events.first(), result.flushed_events.last())
{
self.reader.extend_active_range(first.seq, last.seq);
}
Ok(result)
}
pub fn broadcast_result(&self, result: &SyncResult) {
result.flushed_events.iter().for_each(|e| {
let _ = self.broadcast_tx.send(valid_event_to_raw(e));
});
}
pub fn sync_and_broadcast(&self) -> io::Result<SyncResult> {
let result = self.sync_data()?;
self.broadcast_result(&result);
Ok(result)
}
pub fn get_events_since(
&self,
cursor: EventSequence,
limit: usize,
) -> io::Result<Vec<SequencedEvent>> {
let raw_events = self.reader.read_events_from(cursor, limit)?;
raw_events
.iter()
.map(|raw| {
let payload = decode_payload(&raw.payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
to_sequenced_event(raw, &payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
})
.collect()
}
pub fn get_events_with_mutations_since(
&self,
cursor: EventSequence,
limit: usize,
) -> io::Result<Vec<EventWithMutations>> {
let raw_events = self.reader.read_events_from(cursor, limit)?;
raw_events
.iter()
.map(|raw| {
let payload = decode_payload(&raw.payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let mutation_set = payload.mutation_set.clone();
let event = to_sequenced_event(raw, &payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
Ok(EventWithMutations {
event,
mutation_set,
})
})
.collect()
}
pub fn get_event(&self, seq: EventSequence) -> io::Result<Option<SequencedEvent>> {
self.reader.read_event_at(seq)?.map_or(Ok(None), |raw| {
let payload = decode_payload(&raw.payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let event = to_sequenced_event(&raw, &payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
Ok(Some(event))
})
}
pub fn max_seq(&self) -> EventSequence {
let raw = self.synced_seq.load(Ordering::Acquire);
match raw {
0 => EventSequence::BEFORE_ALL,
n => EventSequence::new(n),
}
}
pub fn subscribe(&self) -> broadcast::Receiver<RawEvent> {
self.broadcast_tx.subscribe()
}
pub fn maybe_rotate(&self) -> io::Result<bool> {
let (sealed_id, new_active_id) = {
let mut writer = self.writer.lock();
match writer.rotate_if_needed()? {
None => return Ok(false),
Some(sealed_id) => (sealed_id, writer.active_segment_id()),
}
};
self.reader.on_segment_rotated(sealed_id, new_active_id)?;
Ok(true)
}
pub fn run_retention(&self, max_age: Duration) -> io::Result<usize> {
let max_age_us = u64::try_from(max_age.as_micros()).unwrap_or(u64::MAX);
let cutoff_us = TimestampMicros::now().raw().saturating_sub(max_age_us);
let active_id = self.writer.lock().active_segment_id();
let segments = self.manager.list_segments()?;
let deleted = segments
.iter()
.take_while(|&&id| id != active_id)
.filter(|&&id| {
self.reader
.load_index(id)
.ok()
.and_then(|idx| idx.last_seq())
.and_then(|seq| {
self.reader
.read_event_at(seq)
.ok()
.flatten()
.map(|e| e.timestamp.raw() < cutoff_us)
})
.unwrap_or(false)
})
.copied()
.collect::<Vec<_>>();
deleted.iter().try_for_each(|&id| -> io::Result<()> {
self.manager.delete_segment(id)?;
self.reader.invalidate_index(id);
self.reader.invalidate_mmap(id);
Ok(())
})?;
if !deleted.is_empty() {
self.reader.refresh_segment_ranges()?;
}
Ok(deleted.len())
}
pub fn segment_count(&self) -> usize {
self.manager.list_segments().map_or(0, |s| s.len())
}
pub fn disk_usage(&self) -> io::Result<u64> {
let segments = self.manager.list_segments()?;
segments.iter().try_fold(0u64, |acc, &id| {
let fd = self.manager.open_for_read(id)?;
let size = self.manager.io().file_size(fd)?;
Ok(acc.saturating_add(size))
})
}
pub fn shutdown(&self) -> io::Result<()> {
self.writer.lock().shutdown()
}
pub fn subscriber(&self, start_seq: EventSequence) -> EventLogSubscriber<S> {
EventLogSubscriber::new(
self.broadcast_tx.subscribe(),
Arc::clone(&self.reader),
start_seq,
)
}
pub fn reader(&self) -> &EventLogReader<S> {
&self.reader
}
pub fn manager(&self) -> &Arc<SegmentManager<S>> {
&self.manager
}
pub fn consecutive_sync_failures(&self) -> u32 {
self.consecutive_sync_failures.load(Ordering::Relaxed)
}
}
impl<S: StorageIO + Send + Sync> PostBlockstoreHook for EventLog<S> {
fn on_blocks_synced(&self, _proof: &BlocksSynced) -> io::Result<()> {
match self.sync_and_broadcast() {
Ok(_) => {
self.consecutive_sync_failures.store(0, Ordering::Relaxed);
if let Err(e) = self.maybe_rotate() {
warn!(error = %e, "eventlog rotation deferred");
}
Ok(())
}
Err(e) => {
let count = self
.consecutive_sync_failures
.fetch_add(1, Ordering::Relaxed)
.saturating_add(1);
warn!(
error = %e,
consecutive_failures = count,
"eventlog sync failed after blockstore commit"
);
Err(e)
}
}
}
}
pub struct EventLogSubscriber<S: StorageIO> {
rx: broadcast::Receiver<RawEvent>,
last_seen: EventSequence,
reader: Arc<EventLogReader<S>>,
backfill_buffer: VecDeque<RawEvent>,
consecutive_lags: u32,
last_lag_time: Option<Instant>,
}
const MAX_CONSECUTIVE_LAGS_BEFORE_WARN: u32 = 3;
const LAG_WINDOW: Duration = Duration::from_secs(10);
const BACKFILL_BATCH_SIZE: usize = 1024;
impl<S: StorageIO> EventLogSubscriber<S> {
pub fn new(
rx: broadcast::Receiver<RawEvent>,
reader: Arc<EventLogReader<S>>,
start_seq: EventSequence,
) -> Self {
Self {
rx,
last_seen: start_seq,
reader,
backfill_buffer: VecDeque::new(),
consecutive_lags: 0,
last_lag_time: None,
}
}
pub async fn next(&mut self) -> Option<RawEvent> {
loop {
if let Some(event) = self.backfill_buffer.pop_front() {
self.last_seen = event.seq;
self.consecutive_lags = 0;
return Some(event);
}
match self.rx.recv().await {
Ok(event) if event.seq > self.last_seen => {
self.last_seen = event.seq;
self.consecutive_lags = 0;
return Some(event);
}
Ok(_) => continue,
Err(broadcast::error::RecvError::Lagged(n)) => {
warn!(
lagged = n,
last_seen = %self.last_seen,
"subscriber lagged, backfilling from disk"
);
self.track_lag();
match self.fill_backfill_buffer() {
Ok(()) => continue,
Err(e) => {
warn!(error = %e, "backfill failed");
return None;
}
}
}
Err(broadcast::error::RecvError::Closed) => return None,
}
}
}
fn fill_backfill_buffer(&mut self) -> io::Result<()> {
let events = self
.reader
.read_events_from(self.last_seen, BACKFILL_BATCH_SIZE)?;
events.into_iter().for_each(|event| {
self.backfill_buffer.push_back(event);
});
Ok(())
}
fn track_lag(&mut self) {
let now = Instant::now();
let in_window = self
.last_lag_time
.is_some_and(|t| now.duration_since(t) < LAG_WINDOW);
if in_window {
self.consecutive_lags = self.consecutive_lags.saturating_add(1);
} else {
self.consecutive_lags = 1;
}
self.last_lag_time = Some(now);
if self.consecutive_lags >= MAX_CONSECUTIVE_LAGS_BEFORE_WARN {
warn!(
consecutive_lags = self.consecutive_lags,
last_seen = %self.last_seen,
"subscriber repeatedly falling behind"
);
}
}
pub fn last_seen(&self) -> EventSequence {
self.last_seen
}
}
fn valid_event_to_raw(e: &ValidEvent) -> RawEvent {
RawEvent {
seq: e.seq,
timestamp: e.timestamp,
did_hash: e.did_hash,
event_type: e.event_type,
payload: bytes::Bytes::from(e.payload.clone()),
}
}
fn repo_event_type_to_tag(event_type: RepoEventType) -> EventTypeTag {
match event_type {
RepoEventType::Commit => EventTypeTag::COMMIT,
RepoEventType::Identity => EventTypeTag::IDENTITY,
RepoEventType::Account => EventTypeTag::ACCOUNT,
RepoEventType::Sync => EventTypeTag::SYNC,
}
}
@@ -0,0 +1,36 @@
use std::sync::Arc;
use async_trait::async_trait;
use tranquil_db_traits::{DbError, RepoEventNotifier, RepoEventReceiver};
use super::{EventLog, EventLogSubscriber, EventSequence};
use crate::io::StorageIO;
pub struct EventLogNotifier<S: StorageIO> {
log: Arc<EventLog<S>>,
}
impl<S: StorageIO> EventLogNotifier<S> {
pub fn new(log: Arc<EventLog<S>>) -> Self {
Self { log }
}
}
#[async_trait]
impl<S: StorageIO + 'static> RepoEventNotifier for EventLogNotifier<S> {
async fn subscribe(&self) -> Result<Box<dyn RepoEventReceiver>, DbError> {
let subscriber = self.log.subscriber(EventSequence::BEFORE_ALL);
Ok(Box::new(EventLogEventReceiver { subscriber }))
}
}
struct EventLogEventReceiver<S: StorageIO> {
subscriber: EventLogSubscriber<S>,
}
#[async_trait]
impl<S: StorageIO + 'static> RepoEventReceiver for EventLogEventReceiver<S> {
async fn recv(&mut self) -> Option<i64> {
self.subscriber.next().await.map(|event| event.seq.as_i64())
}
}
@@ -0,0 +1,461 @@
use serde::{Deserialize, Serialize};
use tranquil_db_traits::{AccountStatus, SequenceNumber, SequencedEvent};
use tranquil_types::{CidLink, Did, Handle};
use crate::eventlog::reader::RawEvent;
use crate::eventlog::types::MAX_EVENT_PAYLOAD;
const PAYLOAD_VERSION: u8 = 1;
const LARGE_PAYLOAD_WARNING_THRESHOLD: usize = 1024 * 1024;
const CID_BYTE_LEN: usize = 36;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventPayload {
pub did: String,
pub commit_cid: Option<Vec<u8>>,
pub prev_cid: Option<Vec<u8>>,
pub prev_data_cid: Option<Vec<u8>>,
pub ops: Option<Vec<u8>>,
pub blobs: Option<Vec<String>>,
pub blocks_cids: Option<Vec<String>>,
pub handle: Option<String>,
pub active: Option<bool>,
pub status: Option<u8>,
pub rev: Option<String>,
pub mutation_set: Option<Vec<u8>>,
}
#[derive(Debug, thiserror::Error)]
pub enum PayloadError {
#[error("payload too large: {size} bytes exceeds max {max}")]
TooLarge { size: usize, max: usize },
#[error("deserialization failed: {0}")]
DeserializeFailed(postcard::Error),
#[error("unknown payload version: {0}")]
UnknownVersion(u8),
#[error("invalid DID in payload: {0}")]
InvalidDid(String),
#[error("invalid timestamp: {0}")]
InvalidTimestamp(u64),
#[error("invalid ops JSON in payload: {0}")]
InvalidOps(serde_json::Error),
#[error("invalid handle in payload: {0}")]
InvalidHandle(String),
#[error("invalid CID length: got {got}, expected {expected}")]
InvalidCidLength { got: usize, expected: usize },
}
fn cid_link_to_bytes(cid: &CidLink) -> Option<Vec<u8>> {
let c = cid.to_cid()?;
let raw = c.to_bytes();
(raw.len() == CID_BYTE_LEN).then_some(raw)
}
fn bytes_to_cid_link(bytes: &[u8]) -> Result<Option<CidLink>, PayloadError> {
if bytes.len() != CID_BYTE_LEN {
return Err(PayloadError::InvalidCidLength {
got: bytes.len(),
expected: CID_BYTE_LEN,
});
}
Ok(cid::Cid::read_bytes(bytes)
.ok()
.map(|c| CidLink::from_cid(&c)))
}
fn account_status_to_u8(status: &AccountStatus) -> u8 {
match status {
AccountStatus::Active => 0,
AccountStatus::Takendown => 1,
AccountStatus::Suspended => 2,
AccountStatus::Deactivated => 3,
AccountStatus::Deleted => 4,
}
}
fn u8_to_account_status(tag: u8) -> Option<AccountStatus> {
match tag {
0 => Some(AccountStatus::Active),
1 => Some(AccountStatus::Takendown),
2 => Some(AccountStatus::Suspended),
3 => Some(AccountStatus::Deactivated),
4 => Some(AccountStatus::Deleted),
_ => None,
}
}
pub fn encode_payload(event: &SequencedEvent) -> Vec<u8> {
encode_payload_with_mutations(event, None)
}
pub fn encode_payload_with_mutations(
event: &SequencedEvent,
mutation_set: Option<&[u8]>,
) -> Vec<u8> {
let ops_bytes = event
.ops
.as_ref()
.map(|v| serde_json::to_vec(v).expect("serde_json::Value always serializes"));
let payload = EventPayload {
did: event.did.as_str().to_owned(),
commit_cid: event.commit_cid.as_ref().and_then(cid_link_to_bytes),
prev_cid: event.prev_cid.as_ref().and_then(cid_link_to_bytes),
prev_data_cid: event.prev_data_cid.as_ref().and_then(cid_link_to_bytes),
ops: ops_bytes,
blobs: event.blobs.clone(),
blocks_cids: event.blocks_cids.clone(),
handle: event
.handle
.as_ref()
.map(|h: &Handle| h.as_str().to_owned()),
active: event.active,
status: event.status.as_ref().map(account_status_to_u8),
rev: event.rev.clone(),
mutation_set: mutation_set.map(|b| b.to_vec()),
};
let body = postcard::to_allocvec(&payload).expect("EventPayload serialization is infallible");
if body.len() > LARGE_PAYLOAD_WARNING_THRESHOLD {
tracing::warn!(
size = body.len(),
did = %event.did,
"unusually large event payload"
);
}
let mut buf = Vec::with_capacity(1 + body.len());
buf.push(PAYLOAD_VERSION);
buf.extend_from_slice(&body);
buf
}
pub fn decode_payload(bytes: &[u8]) -> Result<EventPayload, PayloadError> {
let (&version, body) = bytes.split_first().ok_or(PayloadError::DeserializeFailed(
postcard::Error::DeserializeUnexpectedEnd,
))?;
if version != PAYLOAD_VERSION {
return Err(PayloadError::UnknownVersion(version));
}
postcard::from_bytes(body).map_err(PayloadError::DeserializeFailed)
}
pub fn validate_payload_size(payload: &[u8]) -> Result<(), PayloadError> {
let max = MAX_EVENT_PAYLOAD as usize;
if payload.len() > max {
return Err(PayloadError::TooLarge {
size: payload.len(),
max,
});
}
Ok(())
}
pub fn to_sequenced_event(
raw: &RawEvent,
payload: &EventPayload,
) -> Result<SequencedEvent, PayloadError> {
let timestamp_secs = raw.timestamp.raw() / 1_000_000;
let timestamp_secs_i64 = i64::try_from(timestamp_secs)
.map_err(|_| PayloadError::InvalidTimestamp(raw.timestamp.raw()))?;
let timestamp_subsec_us =
u32::try_from(raw.timestamp.raw() % 1_000_000).expect("modulo 1M always fits u32");
let created_at =
chrono::DateTime::from_timestamp(timestamp_secs_i64, timestamp_subsec_us * 1_000)
.unwrap_or_default();
let did = Did::new(&payload.did).map_err(|_| PayloadError::InvalidDid(payload.did.clone()))?;
let ops = payload
.ops
.as_ref()
.map(|bytes| serde_json::from_slice(bytes))
.transpose()
.map_err(PayloadError::InvalidOps)?;
let handle = payload
.handle
.as_ref()
.map(|h| Handle::new(h.as_str()).map_err(|_| PayloadError::InvalidHandle(h.clone())))
.transpose()?;
Ok(SequencedEvent {
seq: SequenceNumber::from_raw(raw.seq.as_i64()),
did,
created_at,
event_type: raw.event_type.to_repo_event_type(),
commit_cid: payload
.commit_cid
.as_deref()
.map(bytes_to_cid_link)
.transpose()?
.flatten(),
prev_cid: payload
.prev_cid
.as_deref()
.map(bytes_to_cid_link)
.transpose()?
.flatten(),
prev_data_cid: payload
.prev_data_cid
.as_deref()
.map(bytes_to_cid_link)
.transpose()?
.flatten(),
ops,
blobs: payload.blobs.clone(),
blocks_cids: payload.blocks_cids.clone(),
handle,
active: payload.active,
status: payload.status.and_then(u8_to_account_status),
rev: payload.rev.clone(),
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::eventlog::types::{DidHash, EventSequence, EventTypeTag, TimestampMicros};
use bytes::Bytes;
use sha2::Digest;
use tranquil_db_traits::RepoEventType;
fn test_did() -> Did {
Did::new("did:plc:testuser1234567890abcdef").unwrap()
}
fn test_cid_link() -> CidLink {
let hash = sha2::Digest::finalize(sha2::Sha256::new());
let mh = multihash::Multihash::<64>::wrap(0x12, &hash).unwrap();
let c = cid::Cid::new_v1(0x71, mh);
CidLink::from_cid(&c)
}
#[test]
fn round_trip_minimal_payload() {
let event = SequencedEvent {
seq: SequenceNumber::from_raw(42),
did: test_did(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Account,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: None,
active: Some(true),
status: Some(AccountStatus::Active),
rev: None,
};
let encoded = encode_payload(&event);
assert_eq!(encoded[0], PAYLOAD_VERSION);
let decoded = decode_payload(&encoded).unwrap();
assert_eq!(decoded.did, event.did.as_str());
assert_eq!(decoded.active, Some(true));
assert_eq!(decoded.status, Some(0));
assert!(decoded.commit_cid.is_none());
}
#[test]
fn round_trip_full_commit_payload() {
let cid = test_cid_link();
let ops = serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}]);
let event = SequencedEvent {
seq: SequenceNumber::from_raw(100),
did: test_did(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: Some(cid.clone()),
prev_data_cid: Some(cid.clone()),
ops: Some(ops.clone()),
blobs: Some(vec!["bafkreibtest".to_owned()]),
blocks_cids: Some(vec!["bafyreiblock".to_owned()]),
handle: Some(Handle::new("test.bsky.social").unwrap()),
active: None,
status: None,
rev: Some("rev123".to_owned()),
};
let encoded = encode_payload(&event);
let decoded = decode_payload(&encoded).unwrap();
let raw = RawEvent {
seq: EventSequence::new(100),
timestamp: TimestampMicros::now(),
did_hash: DidHash::from_did(event.did.as_str()),
event_type: EventTypeTag::COMMIT,
payload: Bytes::from(encoded),
};
let reconstructed = to_sequenced_event(&raw, &decoded).unwrap();
assert_eq!(reconstructed.did.as_str(), event.did.as_str());
assert_eq!(reconstructed.commit_cid, event.commit_cid);
assert_eq!(reconstructed.prev_cid, event.prev_cid);
assert_eq!(reconstructed.prev_data_cid, event.prev_data_cid);
assert_eq!(reconstructed.blobs, event.blobs);
assert_eq!(reconstructed.blocks_cids, event.blocks_cids);
assert_eq!(
reconstructed.handle.as_ref().map(|h: &Handle| h.as_str()),
event.handle.as_ref().map(|h: &Handle| h.as_str())
);
assert_eq!(reconstructed.rev, event.rev);
assert_eq!(reconstructed.event_type, RepoEventType::Commit);
let reconstructed_ops = reconstructed.ops.unwrap();
assert_eq!(reconstructed_ops, ops);
}
#[test]
fn unknown_version_rejected() {
let mut encoded = encode_payload(&SequencedEvent {
seq: SequenceNumber::from_raw(1),
did: test_did(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Identity,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: None,
active: None,
status: None,
rev: None,
});
encoded[0] = 99;
match decode_payload(&encoded) {
Err(PayloadError::UnknownVersion(99)) => {}
other => panic!("expected UnknownVersion(99), got {other:?}"),
}
}
#[test]
fn empty_payload_rejected() {
match decode_payload(&[]) {
Err(PayloadError::DeserializeFailed(_)) => {}
other => panic!("expected DeserializeFailed, got {other:?}"),
}
}
#[test]
fn validate_payload_size_accepts_within_limit() {
let data = vec![0u8; MAX_EVENT_PAYLOAD as usize];
assert!(validate_payload_size(&data).is_ok());
}
#[test]
fn validate_payload_size_rejects_oversized() {
let data = vec![0u8; MAX_EVENT_PAYLOAD as usize + 1];
match validate_payload_size(&data) {
Err(PayloadError::TooLarge { size, max }) => {
assert_eq!(size, MAX_EVENT_PAYLOAD as usize + 1);
assert_eq!(max, MAX_EVENT_PAYLOAD as usize);
}
other => panic!("expected TooLarge, got {other:?}"),
}
}
#[test]
fn account_status_round_trip() {
let statuses = [
AccountStatus::Active,
AccountStatus::Takendown,
AccountStatus::Suspended,
AccountStatus::Deactivated,
AccountStatus::Deleted,
];
statuses.iter().for_each(|status| {
let tag = account_status_to_u8(status);
let recovered = u8_to_account_status(tag).unwrap();
assert_eq!(&recovered, status);
});
}
#[test]
fn invalid_account_status_returns_none() {
assert!(u8_to_account_status(255).is_none());
}
#[test]
fn cid_bytes_round_trip() {
let cid = test_cid_link();
let bytes = cid_link_to_bytes(&cid).unwrap();
assert_eq!(bytes.len(), CID_BYTE_LEN);
let recovered = bytes_to_cid_link(&bytes).unwrap().unwrap();
assert_eq!(cid, recovered);
}
#[test]
fn cid_bytes_wrong_length_rejected() {
let short = vec![0u8; 10];
match bytes_to_cid_link(&short) {
Err(PayloadError::InvalidCidLength {
got: 10,
expected: 36,
}) => {}
other => panic!("expected InvalidCidLength, got {other:?}"),
}
}
#[test]
fn event_type_tag_mapping() {
assert_eq!(
EventTypeTag::COMMIT.to_repo_event_type(),
RepoEventType::Commit
);
assert_eq!(
EventTypeTag::IDENTITY.to_repo_event_type(),
RepoEventType::Identity
);
assert_eq!(
EventTypeTag::ACCOUNT.to_repo_event_type(),
RepoEventType::Account
);
assert_eq!(EventTypeTag::SYNC.to_repo_event_type(), RepoEventType::Sync);
}
#[test]
fn timestamp_microseconds_preserved() {
let us = 1_700_000_000_123_456u64;
let raw = RawEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(us),
did_hash: DidHash::from_did("did:plc:test"),
event_type: EventTypeTag::COMMIT,
payload: Bytes::new(),
};
let payload = EventPayload {
did: "did:plc:testuser1234567890abcdef".to_owned(),
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: None,
active: None,
status: None,
rev: None,
mutation_set: None,
};
let event = to_sequenced_event(&raw, &payload).unwrap();
let recovered_us = u64::try_from(event.created_at.timestamp_micros()).unwrap();
assert_eq!(recovered_us, us);
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,921 @@
use std::io;
use crate::io::{FileId, StorageIO};
use super::types::{
DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset,
TimestampMicros,
};
pub const SEGMENT_MAGIC: [u8; 4] = *b"TQEV";
pub const SEGMENT_FORMAT_VERSION: u8 = 1;
pub const SEGMENT_HEADER_SIZE: usize = 5;
pub const EVENT_HEADER_SIZE: usize = 8 + 8 + 4 + 1 + 4;
pub const EVENT_RECORD_OVERHEAD: usize = EVENT_HEADER_SIZE + 4;
#[must_use]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ValidEvent {
pub seq: EventSequence,
pub timestamp: TimestampMicros,
pub did_hash: DidHash,
pub event_type: EventTypeTag,
pub payload: Vec<u8>,
}
fn event_record_checksum(header: &[u8; EVENT_HEADER_SIZE], payload: &[u8]) -> u32 {
let mut hasher = xxhash_rust::xxh3::Xxh3::new();
hasher.update(header);
hasher.update(payload);
hasher.digest() as u32
}
fn encode_header(event: &ValidEvent, payload_len: u32) -> [u8; EVENT_HEADER_SIZE] {
let mut header = [0u8; EVENT_HEADER_SIZE];
header[0..8].copy_from_slice(&event.seq.raw().to_le_bytes());
header[8..16].copy_from_slice(&event.timestamp.raw().to_le_bytes());
header[16..20].copy_from_slice(&event.did_hash.raw().to_le_bytes());
header[20] = event.event_type.raw();
header[21..25].copy_from_slice(&payload_len.to_le_bytes());
header
}
pub fn encode_event_record<S: StorageIO>(
io: &S,
fd: FileId,
offset: SegmentOffset,
event: &ValidEvent,
) -> io::Result<u64> {
let payload_len = u32::try_from(event.payload.len()).map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidInput,
"event payload exceeds u32::MAX",
)
})?;
if payload_len > MAX_EVENT_PAYLOAD {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"event payload exceeds MAX_EVENT_PAYLOAD",
));
}
let header = encode_header(event, payload_len);
let checksum = event_record_checksum(&header, &event.payload);
let record_size = EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len);
let base = offset.raw();
io.write_all_at(fd, base, &header)?;
io.write_all_at(fd, base + EVENT_HEADER_SIZE as u64, &event.payload)?;
io.write_all_at(
fd,
base + EVENT_HEADER_SIZE as u64 + u64::from(payload_len),
&checksum.to_le_bytes(),
)?;
Ok(record_size)
}
#[must_use]
#[derive(Debug)]
pub enum ReadEventRecord {
Valid {
event: ValidEvent,
next_offset: SegmentOffset,
},
Corrupted {
offset: SegmentOffset,
},
Truncated {
offset: SegmentOffset,
},
}
pub fn decode_event_record<S: StorageIO>(
io: &S,
fd: FileId,
offset: SegmentOffset,
file_size: u64,
) -> io::Result<Option<ReadEventRecord>> {
let raw = offset.raw();
if raw > file_size {
return Ok(Some(ReadEventRecord::Corrupted { offset }));
}
let remaining = file_size - raw;
if remaining == 0 {
return Ok(None);
}
if remaining < EVENT_HEADER_SIZE as u64 {
return Ok(Some(ReadEventRecord::Truncated { offset }));
}
let mut header = [0u8; EVENT_HEADER_SIZE];
io.read_exact_at(fd, raw, &mut header)?;
let seq_raw = u64::from_le_bytes(header[0..8].try_into().unwrap());
if seq_raw == 0 {
return Ok(Some(ReadEventRecord::Corrupted { offset }));
}
let seq = EventSequence::new(seq_raw);
let timestamp = TimestampMicros::new(u64::from_le_bytes(header[8..16].try_into().unwrap()));
let did_hash = DidHash::from_raw(u32::from_le_bytes(header[16..20].try_into().unwrap()));
let event_type_raw = header[20];
let event_type = match EventTypeTag::from_raw(event_type_raw) {
Some(t) => t,
None => return Ok(Some(ReadEventRecord::Corrupted { offset })),
};
let payload_len = u32::from_le_bytes(header[21..25].try_into().unwrap());
if payload_len > MAX_EVENT_PAYLOAD {
return Ok(Some(ReadEventRecord::Corrupted { offset }));
}
let record_size = EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len);
if record_size > remaining {
return Ok(Some(ReadEventRecord::Truncated { offset }));
}
let payload_offset = raw + EVENT_HEADER_SIZE as u64;
let mut payload = vec![0u8; usize::try_from(payload_len).expect("payload_len fits usize")];
io.read_exact_at(fd, payload_offset, &mut payload)?;
let mut checksum_bytes = [0u8; 4];
io.read_exact_at(
fd,
payload_offset + u64::from(payload_len),
&mut checksum_bytes,
)?;
let stored_checksum = u32::from_le_bytes(checksum_bytes);
let computed_checksum = event_record_checksum(&header, &payload);
if stored_checksum != computed_checksum {
return Ok(Some(ReadEventRecord::Corrupted { offset }));
}
let next_offset = offset.advance(record_size);
Ok(Some(ReadEventRecord::Valid {
event: ValidEvent {
seq,
timestamp,
did_hash,
event_type,
payload,
},
next_offset,
}))
}
#[derive(Debug)]
pub enum ValidateEventRecord {
Valid {
seq: EventSequence,
next_offset: SegmentOffset,
},
Corrupted,
Truncated,
}
const CHECKSUM_CHUNK_SIZE: usize = 8 * 1024;
pub fn validate_event_record<S: StorageIO>(
io: &S,
fd: FileId,
offset: SegmentOffset,
file_size: u64,
) -> io::Result<Option<ValidateEventRecord>> {
let raw = offset.raw();
assert!(
raw <= file_size,
"validate offset {raw} past file size {file_size}"
);
let remaining = file_size - raw;
if remaining == 0 {
return Ok(None);
}
if remaining < EVENT_HEADER_SIZE as u64 {
return Ok(Some(ValidateEventRecord::Truncated));
}
let mut header = [0u8; EVENT_HEADER_SIZE];
io.read_exact_at(fd, raw, &mut header)?;
let seq_raw = u64::from_le_bytes(header[0..8].try_into().unwrap());
if seq_raw == 0 {
return Ok(Some(ValidateEventRecord::Corrupted));
}
let seq = EventSequence::new(seq_raw);
let event_type_raw = header[20];
if EventTypeTag::from_raw(event_type_raw).is_none() {
return Ok(Some(ValidateEventRecord::Corrupted));
}
let payload_len = u32::from_le_bytes(header[21..25].try_into().unwrap());
if payload_len > MAX_EVENT_PAYLOAD {
return Ok(Some(ValidateEventRecord::Corrupted));
}
let record_size = EVENT_RECORD_OVERHEAD as u64 + u64::from(payload_len);
if record_size > remaining {
return Ok(Some(ValidateEventRecord::Truncated));
}
let payload_offset = raw + EVENT_HEADER_SIZE as u64;
let mut hasher = xxhash_rust::xxh3::Xxh3::new();
hasher.update(&header);
let mut chunk = [0u8; CHECKSUM_CHUNK_SIZE];
(0..u64::from(payload_len))
.step_by(CHECKSUM_CHUNK_SIZE)
.map(|chunk_start| {
let to_read =
((u64::from(payload_len) - chunk_start) as usize).min(CHECKSUM_CHUNK_SIZE);
(payload_offset + chunk_start, to_read)
})
.try_for_each(|(pos, to_read)| {
io.read_exact_at(fd, pos, &mut chunk[..to_read])?;
hasher.update(&chunk[..to_read]);
Ok::<_, io::Error>(())
})?;
let computed_checksum = hasher.digest() as u32;
let mut checksum_bytes = [0u8; 4];
io.read_exact_at(
fd,
payload_offset + u64::from(payload_len),
&mut checksum_bytes,
)?;
let stored_checksum = u32::from_le_bytes(checksum_bytes);
if stored_checksum != computed_checksum {
return Ok(Some(ValidateEventRecord::Corrupted));
}
let next_offset = offset.advance(record_size);
Ok(Some(ValidateEventRecord::Valid { seq, next_offset }))
}
pub struct SegmentWriter {
fd: FileId,
segment_id: SegmentId,
position: SegmentOffset,
base_seq: EventSequence,
last_seq: Option<EventSequence>,
}
impl SegmentWriter {
pub fn new<S: StorageIO>(
io: &S,
fd: FileId,
segment_id: SegmentId,
base_seq: EventSequence,
) -> io::Result<Self> {
let mut header = [0u8; SEGMENT_HEADER_SIZE];
header[..4].copy_from_slice(&SEGMENT_MAGIC);
header[4] = SEGMENT_FORMAT_VERSION;
io.write_all_at(fd, 0, &header)?;
Ok(Self {
fd,
segment_id,
position: SegmentOffset::new(SEGMENT_HEADER_SIZE as u64),
base_seq,
last_seq: None,
})
}
pub fn resume<S: StorageIO>(
io: &S,
fd: FileId,
segment_id: SegmentId,
position: SegmentOffset,
base_seq: EventSequence,
last_seq: Option<EventSequence>,
) -> Self {
assert!(
position.raw() >= SEGMENT_HEADER_SIZE as u64,
"resume position {position:?} is before header end"
);
#[cfg(debug_assertions)]
{
let mut magic = [0u8; 4];
io.read_exact_at(fd, 0, &mut magic)
.expect("resume: failed to read segment header");
assert_eq!(magic, SEGMENT_MAGIC, "resume: bad segment magic");
}
#[cfg(not(debug_assertions))]
let _ = io;
Self {
fd,
segment_id,
position,
base_seq,
last_seq,
}
}
pub fn append_event<S: StorageIO>(
&mut self,
io: &S,
event: &ValidEvent,
) -> io::Result<SegmentOffset> {
assert!(
self.last_seq.is_none_or(|prev| event.seq > prev),
"non-monotonic sequence: {} after {}",
event.seq,
self.last_seq.unwrap()
);
let record_offset = self.position;
let bytes_written = encode_event_record(io, self.fd, record_offset, event)?;
self.position = self.position.advance(bytes_written);
self.last_seq = Some(event.seq);
Ok(record_offset)
}
pub fn sync<S: StorageIO>(&self, io: &S) -> io::Result<()> {
io.sync(self.fd)
}
pub fn position(&self) -> SegmentOffset {
self.position
}
pub fn segment_id(&self) -> SegmentId {
self.segment_id
}
pub fn base_seq(&self) -> EventSequence {
self.base_seq
}
pub fn fd(&self) -> FileId {
self.fd
}
}
pub struct SegmentReader<'a, S: StorageIO> {
io: &'a S,
fd: FileId,
position: SegmentOffset,
file_size: u64,
}
impl<'a, S: StorageIO> SegmentReader<'a, S> {
pub fn open(io: &'a S, fd: FileId) -> io::Result<Self> {
let file_size = io.file_size(fd)?;
if file_size < SEGMENT_HEADER_SIZE as u64 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"file too small for segment header",
));
}
let mut header = [0u8; SEGMENT_HEADER_SIZE];
io.read_exact_at(fd, 0, &mut header)?;
if header[..SEGMENT_MAGIC.len()] != SEGMENT_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"bad segment magic",
));
}
if header[SEGMENT_MAGIC.len()] != SEGMENT_FORMAT_VERSION {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported segment format version",
));
}
Ok(Self {
io,
fd,
position: SegmentOffset::new(SEGMENT_HEADER_SIZE as u64),
file_size,
})
}
pub fn valid_prefix(self) -> io::Result<Vec<ValidEvent>> {
self.map(|result| {
result.map(|record| match record {
ReadEventRecord::Valid { event, .. } => Some(event),
ReadEventRecord::Corrupted { .. } | ReadEventRecord::Truncated { .. } => None,
})
})
.scan((), |(), result| match result {
Err(e) => Some(Err(e)),
Ok(Some(event)) => Some(Ok(event)),
Ok(None) => None,
})
.collect()
}
pub fn fd(&self) -> FileId {
self.fd
}
pub fn position(&self) -> SegmentOffset {
self.position
}
pub fn file_size(&self) -> u64 {
self.file_size
}
}
impl<S: StorageIO> Iterator for SegmentReader<'_, S> {
type Item = io::Result<ReadEventRecord>;
fn next(&mut self) -> Option<Self::Item> {
match decode_event_record(self.io, self.fd, self.position, self.file_size) {
Err(e) => {
self.position = SegmentOffset::new(self.file_size);
Some(Err(e))
}
Ok(None) => None,
Ok(Some(record)) => {
match &record {
ReadEventRecord::Valid { next_offset, .. } => {
self.position = *next_offset;
}
ReadEventRecord::Corrupted { .. } | ReadEventRecord::Truncated { .. } => {
self.position = SegmentOffset::new(self.file_size);
}
}
Some(Ok(record))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::OpenOptions;
use crate::sim::SimulatedIO;
use proptest::prelude::*;
use std::path::Path;
fn setup() -> (SimulatedIO, FileId) {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read_write())
.unwrap();
(sim, fd)
}
fn test_did_hash(seed: u8) -> DidHash {
DidHash::from_did(&format!("did:plc:test{seed}"))
}
fn test_event(seq: u64, payload: &[u8]) -> ValidEvent {
ValidEvent {
seq: EventSequence::new(seq),
timestamp: TimestampMicros::new(seq * 1_000_000),
did_hash: test_did_hash(seq as u8),
event_type: EventTypeTag::COMMIT,
payload: payload.to_vec(),
}
}
#[test]
fn write_and_read_single_event() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let event = test_event(1, b"test event payload");
let offset = writer.append_event(&sim, &event).unwrap();
writer.sync(&sim).unwrap();
assert_eq!(offset, SegmentOffset::new(SEGMENT_HEADER_SIZE as u64));
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0], event);
}
#[test]
fn write_and_read_multiple_events() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let written: Vec<ValidEvent> = (1u64..=3)
.map(|i| {
let event = test_event(i, format!("event {i}").as_bytes());
writer.append_event(&sim, &event).unwrap();
event
})
.collect();
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events, written);
}
#[test]
fn empty_segment_has_no_events() {
let (sim, fd) = setup();
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert!(events.is_empty());
}
#[test]
fn detects_truncated_event() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer
.append_event(&sim, &test_event(1, b"complete event"))
.unwrap();
writer.sync(&sim).unwrap();
sim.write_all_at(fd, writer.position().raw(), &[1, 2, 3, 4, 5])
.unwrap();
sim.sync(fd).unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let first = reader.next().unwrap().unwrap();
assert!(matches!(first, ReadEventRecord::Valid { .. }));
let second = reader.next().unwrap().unwrap();
assert!(matches!(second, ReadEventRecord::Truncated { .. }));
}
#[test]
fn checksum_detects_corruption() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer
.append_event(&sim, &test_event(1, &vec![0xAA; 256]))
.unwrap();
writer.sync(&sim).unwrap();
let corrupt_offset = SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64 + 128;
sim.write_all_at(fd, corrupt_offset, &[0x00]).unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(matches!(record, ReadEventRecord::Corrupted { .. }));
}
#[test]
fn crash_before_sync_loses_events() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer
.append_event(&sim, &test_event(1, b"synced"))
.unwrap();
writer.sync(&sim).unwrap();
sim.sync_dir(Path::new("/test")).unwrap();
writer
.append_event(&sim, &test_event(2, b"not synced"))
.unwrap();
sim.crash();
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read())
.unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].payload, b"synced");
}
#[test]
fn rejects_oversized_payload() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let result = writer.append_event(
&sim,
&test_event(1, &vec![0u8; MAX_EVENT_PAYLOAD as usize + 1]),
);
assert!(result.is_err());
}
#[test]
fn zero_length_payload_round_trips() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let event = ValidEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(1_000_000),
did_hash: test_did_hash(1),
event_type: EventTypeTag::IDENTITY,
payload: vec![],
};
writer.append_event(&sim, &event).unwrap();
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events, vec![event]);
}
#[test]
fn accepts_exact_max_payload() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let result = writer.append_event(
&sim,
&test_event(1, &vec![0xBB; MAX_EVENT_PAYLOAD as usize]),
);
assert!(result.is_ok());
}
#[test]
fn bad_magic_rejected() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/bad.tqe"), OpenOptions::read_write())
.unwrap();
sim.write_all_at(fd, 0, b"NOPE\x01").unwrap();
let result = SegmentReader::open(&sim, fd);
assert!(result.is_err());
}
#[test]
fn encode_decode_round_trip_at_offset() {
let (sim, fd) = setup();
sim.write_all_at(fd, 0, &[0u8; 100]).unwrap();
let offset = SegmentOffset::new(100);
let event = ValidEvent {
seq: EventSequence::new(42),
timestamp: TimestampMicros::new(9_999_999),
did_hash: test_did_hash(7),
event_type: EventTypeTag::ACCOUNT,
payload: b"round trip test data".to_vec(),
};
let bytes_written = encode_event_record(&sim, fd, offset, &event).unwrap();
let expected_size = EVENT_RECORD_OVERHEAD as u64 + event.payload.len() as u64;
assert_eq!(bytes_written, expected_size);
let file_size = sim.file_size(fd).unwrap();
let record = decode_event_record(&sim, fd, offset, file_size)
.unwrap()
.unwrap();
match record {
ReadEventRecord::Valid { event: decoded, .. } => assert_eq!(decoded, event),
other => panic!("expected Valid, got {other:?}"),
}
}
#[test]
fn resume_writer_continues_at_position() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
writer.append_event(&sim, &test_event(1, b"first")).unwrap();
writer.sync(&sim).unwrap();
let resume_pos = writer.position();
let mut writer2 = SegmentWriter::resume(
&sim,
fd,
SegmentId::new(1),
resume_pos,
EventSequence::new(1),
Some(EventSequence::new(1)),
);
writer2
.append_event(&sim, &test_event(2, b"second"))
.unwrap();
writer2.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 2);
assert_eq!(events[0].payload, b"first");
assert_eq!(events[1].payload, b"second");
}
#[test]
fn all_event_types_round_trip() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let types = [
EventTypeTag::COMMIT,
EventTypeTag::IDENTITY,
EventTypeTag::ACCOUNT,
EventTypeTag::SYNC,
];
types.iter().enumerate().for_each(|(i, &event_type)| {
let event = ValidEvent {
seq: EventSequence::new((i + 1) as u64),
timestamp: TimestampMicros::new(1_000_000),
did_hash: test_did_hash(i as u8),
event_type,
payload: b"payload".to_vec(),
};
writer.append_event(&sim, &event).unwrap();
});
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 4);
events
.iter()
.zip(types.iter())
.for_each(|(event, &expected_type)| {
assert_eq!(event.event_type, expected_type);
});
}
#[test]
fn seq_zero_detected_as_corrupted() {
let (sim, fd) = setup();
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut raw_header = [0u8; EVENT_HEADER_SIZE];
raw_header[0..8].copy_from_slice(&0u64.to_le_bytes());
raw_header[8..16].copy_from_slice(&1_000_000u64.to_le_bytes());
raw_header[16..20].copy_from_slice(&test_did_hash(1).raw().to_le_bytes());
raw_header[20] = EventTypeTag::COMMIT.raw();
raw_header[21..25].copy_from_slice(&5u32.to_le_bytes());
sim.write_all_at(fd, SEGMENT_HEADER_SIZE as u64, &raw_header)
.unwrap();
sim.write_all_at(
fd,
SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64,
b"hello",
)
.unwrap();
sim.write_all_at(
fd,
SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64 + 5,
&[0u8; 4],
)
.unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(matches!(record, ReadEventRecord::Corrupted { .. }));
}
#[test]
fn writer_accessors() {
let (sim, fd) = setup();
let writer =
SegmentWriter::new(&sim, fd, SegmentId::new(7), EventSequence::new(100)).unwrap();
assert_eq!(writer.segment_id(), SegmentId::new(7));
assert_eq!(writer.base_seq(), EventSequence::new(100));
assert_eq!(
writer.position(),
SegmentOffset::new(SEGMENT_HEADER_SIZE as u64)
);
assert_eq!(writer.fd(), fd);
}
fn run_crash_recovery_seed(seed: u64) {
let sim = SimulatedIO::new(seed, crate::FaultConfig::aggressive());
let dir = Path::new("/data");
let _ = sim.mkdir(dir);
let _ = sim.sync_dir(dir);
let written_count =
if let Ok(fd) = sim.open(Path::new("/data/segment.tqe"), OpenOptions::read_write()) {
if let Ok(mut writer) =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1))
{
let count = (1u64..=20).fold(0u64, |count, i| {
let event = ValidEvent {
seq: EventSequence::new(i),
timestamp: TimestampMicros::new(i * 1_000_000),
did_hash: DidHash::from_did(&format!("did:plc:user{i}")),
event_type: EventTypeTag::COMMIT,
payload: vec![i as u8; ((i as usize) + 1) * 10],
};
match writer.append_event(&sim, &event) {
Ok(_) => count + 1,
Err(_) => count,
}
});
let _ = writer.sync(&sim);
count
} else {
0
}
} else {
0
};
let _ = sim.sync_dir(dir);
sim.crash();
if let Ok(fd) = sim.open(Path::new("/data/segment.tqe"), OpenOptions::read())
&& let Ok(reader) = SegmentReader::open(&sim, fd)
{
let recovered: Vec<_> = reader
.map_while(|r| match r {
Ok(ReadEventRecord::Valid { event, .. }) => Some(event),
_ => None,
})
.collect();
assert!(
recovered.len() as u64 <= written_count,
"recovered {} events but only wrote {written_count}",
recovered.len()
);
recovered.windows(2).enumerate().for_each(|(i, pair)| {
assert!(
pair[0].seq < pair[1].seq,
"event {i} seq {} not less than event {} seq {}",
pair[0].seq,
i + 1,
pair[1].seq,
);
});
}
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(2000))]
#[test]
fn sim_crash_recovery_aggressive_faults(seed in 0u64..u64::MAX) {
run_crash_recovery_seed(seed);
}
}
fn run_bit_flip_detection_seed(seed: u64) {
let sim = SimulatedIO::pristine(seed);
let dir = Path::new("/data");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/data/segment.tqe"), OpenOptions::read_write())
.unwrap();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let data_len = ((seed % 256) as usize).max(1);
let event = ValidEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(1_000_000),
did_hash: DidHash::from_did("did:plc:bitflip"),
event_type: EventTypeTag::COMMIT,
payload: vec![0xAA; data_len],
};
writer.append_event(&sim, &event).unwrap();
writer.sync(&sim).unwrap();
let record_start = SEGMENT_HEADER_SIZE as u64;
let record_end = record_start + EVENT_RECORD_OVERHEAD as u64 + data_len as u64;
let flip_pos = record_start + (seed.wrapping_mul(7) % (record_end - record_start));
let flip_bit = (seed.wrapping_mul(13) % 8) as u8;
let mut byte_buf = [0u8; 1];
sim.read_exact_at(fd, flip_pos, &mut byte_buf).unwrap();
byte_buf[0] ^= 1 << flip_bit;
sim.write_all_at(fd, flip_pos, &byte_buf).unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(
!matches!(record, ReadEventRecord::Valid { .. }),
"bit flip at offset {flip_pos} bit {flip_bit} was not detected"
);
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(2000))]
#[test]
fn sim_bit_flip_detected_by_checksum(seed in 0u64..u64::MAX) {
run_bit_flip_detection_seed(seed);
}
}
}
@@ -0,0 +1,666 @@
use std::cell::Cell;
use std::io;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::io::{FileId, OpenOptions, StorageIO};
use crate::record::{RecordReader, RecordWriter};
use super::segment_file::{
SEGMENT_HEADER_SIZE, SEGMENT_MAGIC, ValidateEventRecord, validate_event_record,
};
use super::types::{EventSequence, SegmentOffset};
pub const DEFAULT_INDEX_INTERVAL: usize = 256;
const MAX_INDEX_ENTRIES: usize = 4 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
struct IndexEntry {
seq: EventSequence,
offset: SegmentOffset,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SegmentIndex {
entries: Vec<IndexEntry>,
}
impl SegmentIndex {
pub fn new() -> Self {
Self {
entries: Vec::new(),
}
}
pub fn record(&mut self, seq: EventSequence, offset: SegmentOffset) {
debug_assert!(
self.entries.last().is_none_or(|last| seq > last.seq),
"index entries must be monotonically increasing"
);
self.entries.push(IndexEntry { seq, offset });
}
pub fn lookup(&self, target_seq: EventSequence) -> Option<SegmentOffset> {
let idx = self.entries.partition_point(|e| e.seq <= target_seq);
match idx {
0 => None,
i => Some(self.entries[i - 1].offset),
}
}
pub fn first_seq(&self) -> Option<EventSequence> {
self.entries.first().map(|e| e.seq)
}
pub fn last_seq(&self) -> Option<EventSequence> {
self.entries.last().map(|e| e.seq)
}
pub fn entry_count(&self) -> usize {
self.entries.len()
}
pub fn save<S: StorageIO>(&self, io: &S, path: &Path) -> io::Result<()> {
let tmp_path = path.with_extension("tqi.tmp");
let fd = io.open(&tmp_path, OpenOptions::read_write())?;
let result = (|| {
let serialized = postcard::to_allocvec(&self.entries)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let mut writer = RecordWriter::new(io, fd)?;
writer.append(&serialized)?;
io.truncate(fd, writer.position())?;
writer.sync()?;
Ok(())
})();
if let Err(e) = result {
let _ = io.close(fd);
return Err(e);
}
io.close(fd)?;
io.rename(&tmp_path, path)?;
if let Some(parent) = path.parent() {
io.sync_dir(parent)?;
}
Ok(())
}
pub fn load<S: StorageIO>(io: &S, path: &Path) -> io::Result<Option<Self>> {
let fd = match io.open(path, OpenOptions::read_only_existing()) {
Ok(fd) => fd,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e),
};
let reader = match RecordReader::open(io, fd) {
Ok(r) => r,
Err(e) => {
let _ = io.close(fd);
return Err(e);
}
};
let records = reader.valid_records();
io.close(fd)?;
let payload = records.into_iter().next().ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "index file contains no records")
})?;
const MIN_POSTCARD_ENTRY_BYTES: usize = 2;
if payload.len() / MIN_POSTCARD_ENTRY_BYTES > MAX_INDEX_ENTRIES {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"index payload too large",
));
}
let entries: Vec<IndexEntry> = postcard::from_bytes(&payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
if entries.len() > MAX_INDEX_ENTRIES {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"index contains too many entries",
));
}
let is_sorted = entries.windows(2).all(|pair| pair[0].seq < pair[1].seq);
if !is_sorted {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"index entries not monotonically sorted",
));
}
Ok(Some(Self { entries }))
}
}
impl Default for SegmentIndex {
fn default() -> Self {
Self::new()
}
}
struct ScanState {
index: SegmentIndex,
event_count: usize,
last_seq: Option<EventSequence>,
last_offset: Option<SegmentOffset>,
}
pub fn rebuild_from_segment<S: StorageIO>(
io: &S,
segment_fd: FileId,
index_interval: usize,
) -> io::Result<(SegmentIndex, Option<EventSequence>)> {
assert!(index_interval > 0, "index_interval must be positive");
let file_size = io.file_size(segment_fd)?;
if file_size < SEGMENT_HEADER_SIZE as u64 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"file too small for segment header",
));
}
let mut header = [0u8; SEGMENT_HEADER_SIZE];
io.read_exact_at(segment_fd, 0, &mut header)?;
if header[..SEGMENT_MAGIC.len()] != SEGMENT_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"bad segment magic",
));
}
if header[SEGMENT_MAGIC.len()] != super::segment_file::SEGMENT_FORMAT_VERSION {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported segment format version",
));
}
let current_offset = Cell::new(SegmentOffset::new(SEGMENT_HEADER_SIZE as u64));
let prev_seq: Cell<Option<EventSequence>> = Cell::new(None);
let mut valid_events = std::iter::from_fn(|| {
let offset = current_offset.get();
if offset.raw() >= file_size {
return None;
}
match validate_event_record(io, segment_fd, offset, file_size) {
Err(e) => Some(Err(e)),
Ok(None) => None,
Ok(Some(ValidateEventRecord::Valid { seq, next_offset })) => {
if prev_seq.get().is_some_and(|prev| seq <= prev) {
return None;
}
prev_seq.set(Some(seq));
current_offset.set(next_offset);
Some(Ok((seq, offset)))
}
Ok(Some(ValidateEventRecord::Corrupted | ValidateEventRecord::Truncated)) => None,
}
});
let initial = ScanState {
index: SegmentIndex::new(),
event_count: 0,
last_seq: None,
last_offset: None,
};
let state = valid_events.try_fold(initial, |mut state, record| -> io::Result<ScanState> {
let (seq, record_offset) = record?;
let should_index = state.event_count == 0 || state.event_count % index_interval == 0;
if should_index {
state.index.record(seq, record_offset);
}
state.event_count += 1;
state.last_seq = Some(seq);
state.last_offset = Some(record_offset);
Ok(state)
})?;
let mut index = state.index;
if let (Some(seq), Some(offset)) = (state.last_seq, state.last_offset)
&& index.last_seq() != Some(seq)
{
index.record(seq, offset);
}
let valid_end = current_offset.get().raw();
if valid_end < file_size {
io.truncate(segment_fd, valid_end)?;
io.sync(segment_fd)?;
}
Ok((index, state.last_seq))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::OpenOptions;
use crate::eventlog::segment_file::{
EVENT_HEADER_SIZE, SegmentWriter, ValidEvent, encode_event_record,
};
use crate::eventlog::types::{
DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros,
};
use crate::sim::SimulatedIO;
use std::path::Path;
fn setup() -> (SimulatedIO, FileId) {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read_write())
.unwrap();
(sim, fd)
}
fn test_event(seq: u64, payload: &[u8]) -> ValidEvent {
ValidEvent {
seq: EventSequence::new(seq),
timestamp: TimestampMicros::new(seq * 1_000_000),
did_hash: DidHash::from_did(&format!("did:plc:test{seq}")),
event_type: EventTypeTag::COMMIT,
payload: payload.to_vec(),
}
}
fn write_n_events<S: StorageIO>(
io: &S,
fd: FileId,
count: u64,
) -> Vec<(EventSequence, SegmentOffset)> {
let mut writer =
SegmentWriter::new(io, fd, SegmentId::new(0), EventSequence::new(1)).unwrap();
let offsets: Vec<_> = (1..=count)
.map(|i| {
let event = test_event(i, format!("payload-{i}").as_bytes());
let offset = writer.append_event(io, &event).unwrap();
(event.seq, offset)
})
.collect();
writer.sync(io).unwrap();
offsets
}
#[test]
fn empty_index() {
let index = SegmentIndex::new();
assert_eq!(index.entry_count(), 0);
assert_eq!(index.first_seq(), None);
assert_eq!(index.last_seq(), None);
assert_eq!(index.lookup(EventSequence::new(1)), None);
}
#[test]
fn record_and_lookup_single_entry() {
let mut index = SegmentIndex::new();
index.record(EventSequence::new(10), SegmentOffset::new(100));
assert_eq!(index.entry_count(), 1);
assert_eq!(index.first_seq(), Some(EventSequence::new(10)));
assert_eq!(index.last_seq(), Some(EventSequence::new(10)));
assert_eq!(
index.lookup(EventSequence::new(10)),
Some(SegmentOffset::new(100))
);
assert_eq!(
index.lookup(EventSequence::new(15)),
Some(SegmentOffset::new(100))
);
assert_eq!(index.lookup(EventSequence::new(5)), None);
}
#[test]
fn lookup_returns_floor_entry() {
let mut index = SegmentIndex::new();
index.record(EventSequence::new(1), SegmentOffset::new(100));
index.record(EventSequence::new(100), SegmentOffset::new(5000));
index.record(EventSequence::new(200), SegmentOffset::new(10000));
assert_eq!(
index.lookup(EventSequence::new(1)),
Some(SegmentOffset::new(100))
);
assert_eq!(
index.lookup(EventSequence::new(50)),
Some(SegmentOffset::new(100))
);
assert_eq!(
index.lookup(EventSequence::new(100)),
Some(SegmentOffset::new(5000))
);
assert_eq!(
index.lookup(EventSequence::new(150)),
Some(SegmentOffset::new(5000))
);
assert_eq!(
index.lookup(EventSequence::new(200)),
Some(SegmentOffset::new(10000))
);
assert_eq!(
index.lookup(EventSequence::new(999)),
Some(SegmentOffset::new(10000))
);
}
#[test]
fn lookup_before_first_returns_none() {
let mut index = SegmentIndex::new();
index.record(EventSequence::new(10), SegmentOffset::new(100));
index.record(EventSequence::new(20), SegmentOffset::new(200));
assert_eq!(index.lookup(EventSequence::new(5)), None);
assert_eq!(index.lookup(EventSequence::new(9)), None);
}
#[test]
fn save_and_load_round_trip() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let mut index = SegmentIndex::new();
index.record(EventSequence::new(1), SegmentOffset::new(5));
index.record(EventSequence::new(256), SegmentOffset::new(50000));
index.record(EventSequence::new(512), SegmentOffset::new(100000));
let path = Path::new("/test/00000001.tqi");
index.save(&sim, path).unwrap();
let loaded = SegmentIndex::load(&sim, path).unwrap().unwrap();
assert_eq!(loaded, index);
}
#[test]
fn load_missing_file_returns_none() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let result = SegmentIndex::load(&sim, Path::new("/test/missing.tqi")).unwrap();
assert!(result.is_none());
}
#[test]
fn load_corrupt_file_returns_err() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let path = Path::new("/test/corrupt.tqi");
let fd = sim.open(path, OpenOptions::read_write()).unwrap();
sim.write_all_at(fd, 0, b"TQST\x02garbage_not_valid_postcard")
.unwrap();
sim.sync(fd).unwrap();
sim.close(fd).unwrap();
let result = SegmentIndex::load(&sim, path);
assert!(result.is_err());
}
#[test]
fn save_empty_index_round_trips() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let index = SegmentIndex::new();
let path = Path::new("/test/empty.tqi");
index.save(&sim, path).unwrap();
let loaded = SegmentIndex::load(&sim, path).unwrap().unwrap();
assert_eq!(loaded.entry_count(), 0);
}
#[test]
fn rebuild_empty_segment() {
let (sim, fd) = setup();
SegmentWriter::new(&sim, fd, SegmentId::new(0), EventSequence::new(1)).unwrap();
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(index.entry_count(), 0);
assert_eq!(last_seq, None);
}
#[test]
fn rebuild_single_event() {
let (sim, fd) = setup();
let offsets = write_n_events(&sim, fd, 1);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(1)));
assert_eq!(index.entry_count(), 1);
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.lookup(EventSequence::new(1)), Some(offsets[0].1));
}
#[test]
fn rebuild_indexes_first_and_last() {
let (sim, fd) = setup();
let offsets = write_n_events(&sim, fd, 10);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(10)));
assert_eq!(index.entry_count(), 2);
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(10)));
assert_eq!(index.lookup(EventSequence::new(1)), Some(offsets[0].1));
assert_eq!(index.lookup(EventSequence::new(10)), Some(offsets[9].1));
}
#[test]
fn rebuild_indexes_at_interval() {
let (sim, fd) = setup();
let offsets = write_n_events(&sim, fd, 600);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, 256).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(600)));
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(600)));
assert_eq!(index.entry_count(), 4);
assert_eq!(index.lookup(EventSequence::new(1)), Some(offsets[0].1));
assert_eq!(index.lookup(EventSequence::new(257)), Some(offsets[256].1));
assert_eq!(index.lookup(EventSequence::new(256)), Some(offsets[0].1));
assert_eq!(index.lookup(EventSequence::new(513)), Some(offsets[512].1));
assert_eq!(index.lookup(EventSequence::new(600)), Some(offsets[599].1));
}
#[test]
fn rebuild_truncates_corruption() {
let (sim, fd) = setup();
write_n_events(&sim, fd, 5);
sim.sync(fd).unwrap();
let file_size_before = sim.file_size(fd).unwrap();
sim.write_all_at(fd, file_size_before, b"garbage_trailing_data")
.unwrap();
sim.sync(fd).unwrap();
let file_size_with_garbage = sim.file_size(fd).unwrap();
assert!(file_size_with_garbage > file_size_before);
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(5)));
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
let file_size_after = sim.file_size(fd).unwrap();
assert_eq!(file_size_after, file_size_before);
}
#[test]
fn rebuild_truncates_partial_record() {
let (sim, fd) = setup();
write_n_events(&sim, fd, 3);
sim.sync(fd).unwrap();
let valid_end = sim.file_size(fd).unwrap();
let partial_header = [0u8; EVENT_HEADER_SIZE - 5];
sim.write_all_at(fd, valid_end, &partial_header).unwrap();
sim.sync(fd).unwrap();
let (_, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(3)));
assert_eq!(sim.file_size(fd).unwrap(), valid_end);
}
#[test]
fn rebuild_truncates_at_non_monotonic_seq() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(0), EventSequence::new(1)).unwrap();
let event1 = test_event(1, b"first");
let event2 = test_event(2, b"second");
writer.append_event(&sim, &event1).unwrap();
let offset_after_two = {
writer.append_event(&sim, &event2).unwrap();
writer.position()
};
writer.sync(&sim).unwrap();
let valid_size_before = offset_after_two.raw();
let regressed = ValidEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(3_000_000),
did_hash: DidHash::from_did("did:plc:test3"),
event_type: EventTypeTag::COMMIT,
payload: b"regressed".to_vec(),
};
encode_event_record(&sim, fd, offset_after_two, &regressed).unwrap();
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(2)));
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(2)));
assert_eq!(sim.file_size(fd).unwrap(), valid_size_before);
}
#[test]
fn rebuild_interval_one_indexes_every_event() {
let (sim, fd) = setup();
let offsets = write_n_events(&sim, fd, 10);
sim.sync(fd).unwrap();
let (index, _) = rebuild_from_segment(&sim, fd, 1).unwrap();
assert_eq!(index.entry_count(), 10);
offsets.iter().enumerate().for_each(|(i, (seq, offset))| {
assert_eq!(index.lookup(*seq), Some(*offset), "event {i} lookup failed");
});
}
#[test]
fn rebuild_and_save_load_round_trip() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read_write())
.unwrap();
write_n_events(&sim, fd, 300);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, 256).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(300)));
let index_path = Path::new("/test/00000000.tqi");
index.save(&sim, index_path).unwrap();
let loaded = SegmentIndex::load(&sim, index_path).unwrap().unwrap();
assert_eq!(loaded, index);
assert_eq!(loaded.entry_count(), index.entry_count());
assert_eq!(loaded.first_seq(), index.first_seq());
assert_eq!(loaded.last_seq(), index.last_seq());
}
#[test]
fn save_overwrites_stale_tmp() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let stale_tmp = Path::new("/test/00000000.tqi.tmp");
let stale_fd = sim.open(stale_tmp, OpenOptions::read_write()).unwrap();
sim.write_all_at(stale_fd, 0, b"stale_garbage_from_prior_crash_xxxxxxxxxx")
.unwrap();
sim.sync(stale_fd).unwrap();
sim.close(stale_fd).unwrap();
let mut index = SegmentIndex::new();
index.record(EventSequence::new(1), SegmentOffset::new(5));
let path = Path::new("/test/00000000.tqi");
index.save(&sim, path).unwrap();
let loaded = SegmentIndex::load(&sim, path).unwrap().unwrap();
assert_eq!(loaded, index);
}
#[test]
fn rebuild_bad_magic_returns_err() {
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/bad.tqe"), OpenOptions::read_write())
.unwrap();
sim.write_all_at(fd, 0, b"NOPE\x01").unwrap();
sim.sync(fd).unwrap();
let result = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL);
assert!(result.is_err());
}
#[test]
fn rebuild_no_truncation_when_clean() {
let (sim, fd) = setup();
write_n_events(&sim, fd, 5);
sim.sync(fd).unwrap();
let size_before = sim.file_size(fd).unwrap();
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let size_after = sim.file_size(fd).unwrap();
assert_eq!(size_before, size_after);
}
#[test]
fn lookup_at_before_all_returns_none() {
let mut index = SegmentIndex::new();
index.record(EventSequence::new(1), SegmentOffset::new(5));
assert_eq!(index.lookup(EventSequence::BEFORE_ALL), None);
}
}
+522
View File
@@ -0,0 +1,522 @@
use serde::{Deserialize, Serialize};
use tranquil_db_traits::SequenceNumber;
pub const MAX_EVENT_PAYLOAD: u32 = 4 * 1024 * 1024;
pub const DEFAULT_SEGMENT_SIZE: u64 = 64 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct EventSequence(u64);
impl EventSequence {
pub const BEFORE_ALL: Self = Self(0);
pub fn new(seq: u64) -> Self {
assert!(
seq > 0,
"EventSequence must be positive; use BEFORE_ALL for cursor start"
);
Self(seq)
}
pub fn raw(self) -> u64 {
self.0
}
pub fn next(self) -> Self {
Self(self.0.checked_add(1).expect("EventSequence overflow"))
}
pub fn prev_or_before_all(self) -> Self {
match self.0 {
0 | 1 => Self::BEFORE_ALL,
n => Self(n - 1),
}
}
pub fn as_i64(self) -> i64 {
i64::try_from(self.0).expect("EventSequence exceeds i64::MAX")
}
pub fn from_i64(n: i64) -> Option<Self> {
match u64::try_from(n) {
Ok(0) | Err(_) => None,
Ok(v) => Some(Self(v)),
}
}
pub fn cursor_from_i64(n: i64) -> Option<Self> {
u64::try_from(n).ok().map(Self)
}
}
impl From<EventSequence> for SequenceNumber {
fn from(es: EventSequence) -> Self {
SequenceNumber::from_raw(es.as_i64())
}
}
impl TryFrom<SequenceNumber> for EventSequence {
type Error = &'static str;
fn try_from(seq: SequenceNumber) -> Result<Self, Self::Error> {
let raw = seq.as_i64();
match u64::try_from(raw) {
Ok(0) => Err("SequenceNumber 0 maps to BEFORE_ALL, not a valid EventSequence"),
Ok(v) => Ok(Self(v)),
Err(_) => Err("negative SequenceNumber cannot convert to EventSequence"),
}
}
}
impl std::fmt::Display for EventSequence {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct SegmentId(u32);
impl SegmentId {
pub fn new(id: u32) -> Self {
Self(id)
}
pub fn raw(self) -> u32 {
self.0
}
pub fn next(self) -> Self {
Self(self.0.checked_add(1).expect("SegmentId overflow"))
}
}
impl std::fmt::Display for SegmentId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:08}", self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct SegmentOffset(u64);
impl SegmentOffset {
pub const fn new(offset: u64) -> Self {
Self(offset)
}
pub const fn raw(self) -> u64 {
self.0
}
pub fn advance(self, delta: u64) -> Self {
Self(self.0.checked_add(delta).expect("SegmentOffset overflow"))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct EventLength(u32);
impl EventLength {
pub fn new(length: u32) -> Self {
assert!(
length <= MAX_EVENT_PAYLOAD,
"EventLength {length} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}"
);
Self(length)
}
pub fn raw(self) -> u32 {
self.0
}
pub fn as_u64(self) -> u64 {
u64::from(self.0)
}
}
impl<'de> Deserialize<'de> for EventLength {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let raw = u32::deserialize(deserializer)?;
if raw > MAX_EVENT_PAYLOAD {
return Err(serde::de::Error::custom(format_args!(
"EventLength {raw} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}"
)));
}
Ok(Self(raw))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct DidHash(u32);
impl DidHash {
pub fn from_did(did: &str) -> Self {
Self(xxhash_rust::xxh3::xxh3_64(did.as_bytes()) as u32)
}
pub fn from_raw(hash: u32) -> Self {
Self(hash)
}
pub fn raw(self) -> u32 {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct EventTypeTag(u8);
impl EventTypeTag {
pub const COMMIT: Self = Self(1);
pub const IDENTITY: Self = Self(2);
pub const ACCOUNT: Self = Self(3);
pub const SYNC: Self = Self(4);
pub fn from_raw(tag: u8) -> Option<Self> {
match tag {
1..=4 => Some(Self(tag)),
_ => None,
}
}
pub fn raw(self) -> u8 {
self.0
}
pub fn to_repo_event_type(self) -> tranquil_db_traits::RepoEventType {
match self {
Self::COMMIT => tranquil_db_traits::RepoEventType::Commit,
Self::IDENTITY => tranquil_db_traits::RepoEventType::Identity,
Self::ACCOUNT => tranquil_db_traits::RepoEventType::Account,
Self::SYNC => tranquil_db_traits::RepoEventType::Sync,
_ => unreachable!("EventTypeTag invariant guarantees valid discriminant"),
}
}
}
impl<'de> Deserialize<'de> for EventTypeTag {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let raw = u8::deserialize(deserializer)?;
Self::from_raw(raw)
.ok_or_else(|| serde::de::Error::custom(format_args!("invalid EventTypeTag: {raw}")))
}
}
impl std::fmt::Display for EventTypeTag {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match *self {
Self::COMMIT => write!(f, "Commit"),
Self::IDENTITY => write!(f, "Identity"),
Self::ACCOUNT => write!(f, "Account"),
Self::SYNC => write!(f, "Sync"),
_ => unreachable!("EventTypeTag invariant violated: raw value {}", self.0),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct TimestampMicros(u64);
impl TimestampMicros {
pub fn new(us: u64) -> Self {
Self(us)
}
pub fn raw(self) -> u64 {
self.0
}
pub fn now() -> Self {
let duration = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system clock before unix epoch");
Self(
duration
.as_secs()
.saturating_mul(1_000_000)
.saturating_add(u64::from(duration.subsec_micros())),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn event_sequence_lifecycle() {
let seq = EventSequence::new(1);
assert_eq!(seq.raw(), 1);
assert_eq!(seq.next(), EventSequence::new(2));
assert_eq!(seq.as_i64(), 1);
}
#[test]
fn event_sequence_before_all() {
assert_eq!(EventSequence::BEFORE_ALL.raw(), 0);
}
#[test]
fn event_sequence_prev_or_before_all() {
assert_eq!(
EventSequence::BEFORE_ALL.prev_or_before_all(),
EventSequence::BEFORE_ALL
);
assert_eq!(
EventSequence::new(1).prev_or_before_all(),
EventSequence::BEFORE_ALL
);
assert_eq!(
EventSequence::new(2).prev_or_before_all(),
EventSequence::new(1)
);
assert_eq!(
EventSequence::new(100).prev_or_before_all(),
EventSequence::new(99)
);
}
#[test]
#[should_panic(expected = "EventSequence must be positive")]
fn event_sequence_zero_panics() {
EventSequence::new(0);
}
#[test]
fn event_sequence_i64_round_trip() {
let seq = EventSequence::new(42);
let as_i64 = seq.as_i64();
assert_eq!(EventSequence::from_i64(as_i64), Some(seq));
}
#[test]
fn event_sequence_from_i64_rejects_zero_and_negative() {
assert_eq!(EventSequence::from_i64(0), None);
assert_eq!(EventSequence::from_i64(-1), None);
}
#[test]
fn event_sequence_cursor_from_i64_allows_zero() {
assert_eq!(
EventSequence::cursor_from_i64(0),
Some(EventSequence::BEFORE_ALL)
);
assert_eq!(
EventSequence::cursor_from_i64(1),
Some(EventSequence::new(1))
);
assert_eq!(EventSequence::cursor_from_i64(-1), None);
}
#[test]
#[should_panic(expected = "EventSequence overflow")]
fn event_sequence_overflow_panics() {
EventSequence::new(u64::MAX).next();
}
#[test]
fn segment_id_display_zero_padded() {
assert_eq!(SegmentId::new(0).to_string(), "00000000");
assert_eq!(SegmentId::new(1).to_string(), "00000001");
assert_eq!(SegmentId::new(99999999).to_string(), "99999999");
}
#[test]
fn segment_id_next_increments() {
assert_eq!(SegmentId::new(0).next(), SegmentId::new(1));
assert_eq!(SegmentId::new(99).next(), SegmentId::new(100));
}
#[test]
#[should_panic(expected = "SegmentId overflow")]
fn segment_id_overflow_panics() {
SegmentId::new(u32::MAX).next();
}
#[test]
fn segment_offset_advance() {
let offset = SegmentOffset::new(100);
assert_eq!(offset.advance(50), SegmentOffset::new(150));
}
#[test]
#[should_panic(expected = "SegmentOffset overflow")]
fn segment_offset_overflow_panics() {
SegmentOffset::new(u64::MAX).advance(1);
}
#[test]
fn event_length_valid() {
let len = EventLength::new(1024);
assert_eq!(len.raw(), 1024);
assert_eq!(len.as_u64(), 1024);
}
#[test]
fn event_length_max_accepted() {
let len = EventLength::new(MAX_EVENT_PAYLOAD);
assert_eq!(len.raw(), MAX_EVENT_PAYLOAD);
}
#[test]
#[should_panic(expected = "exceeds MAX_EVENT_PAYLOAD")]
fn event_length_overflow_panics() {
EventLength::new(MAX_EVENT_PAYLOAD + 1);
}
#[test]
fn did_hash_deterministic() {
let hash1 = DidHash::from_did("did:plc:abc123");
let hash2 = DidHash::from_did("did:plc:abc123");
assert_eq!(hash1, hash2);
}
#[test]
fn did_hash_different_dids_differ() {
let hash1 = DidHash::from_did("did:plc:abc123");
let hash2 = DidHash::from_did("did:plc:xyz789");
assert_ne!(hash1, hash2);
}
#[test]
fn event_type_tag_known_variants() {
assert_eq!(EventTypeTag::COMMIT.raw(), 1);
assert_eq!(EventTypeTag::IDENTITY.raw(), 2);
assert_eq!(EventTypeTag::ACCOUNT.raw(), 3);
assert_eq!(EventTypeTag::SYNC.raw(), 4);
}
#[test]
fn event_type_tag_from_raw_valid() {
assert_eq!(EventTypeTag::from_raw(1), Some(EventTypeTag::COMMIT));
assert_eq!(EventTypeTag::from_raw(2), Some(EventTypeTag::IDENTITY));
assert_eq!(EventTypeTag::from_raw(3), Some(EventTypeTag::ACCOUNT));
assert_eq!(EventTypeTag::from_raw(4), Some(EventTypeTag::SYNC));
}
#[test]
fn event_type_tag_from_raw_invalid() {
assert_eq!(EventTypeTag::from_raw(0), None);
assert_eq!(EventTypeTag::from_raw(5), None);
assert_eq!(EventTypeTag::from_raw(255), None);
}
#[test]
fn event_type_tag_display() {
assert_eq!(EventTypeTag::COMMIT.to_string(), "Commit");
assert_eq!(EventTypeTag::IDENTITY.to_string(), "Identity");
assert_eq!(EventTypeTag::ACCOUNT.to_string(), "Account");
assert_eq!(EventTypeTag::SYNC.to_string(), "Sync");
}
#[test]
fn timestamp_micros_round_trip() {
let ts = TimestampMicros::new(1_700_000_000_000_000);
assert_eq!(ts.raw(), 1_700_000_000_000_000);
}
#[test]
fn timestamp_micros_now_is_reasonable() {
let ts = TimestampMicros::now();
assert!(ts.raw() > 1_700_000_000_000_000);
}
#[test]
fn postcard_round_trip_event_sequence() {
let seq = EventSequence::new(42);
let bytes = postcard::to_allocvec(&seq).unwrap();
let decoded: EventSequence = postcard::from_bytes(&bytes).unwrap();
assert_eq!(seq, decoded);
}
#[test]
fn postcard_round_trip_segment_id() {
let id = SegmentId::new(7);
let bytes = postcard::to_allocvec(&id).unwrap();
let decoded: SegmentId = postcard::from_bytes(&bytes).unwrap();
assert_eq!(id, decoded);
}
#[test]
fn postcard_round_trip_did_hash() {
let hash = DidHash::from_did("did:plc:test");
let bytes = postcard::to_allocvec(&hash).unwrap();
let decoded: DidHash = postcard::from_bytes(&bytes).unwrap();
assert_eq!(hash, decoded);
}
#[test]
fn postcard_round_trip_event_type_tag() {
let tag = EventTypeTag::COMMIT;
let bytes = postcard::to_allocvec(&tag).unwrap();
let decoded: EventTypeTag = postcard::from_bytes(&bytes).unwrap();
assert_eq!(tag, decoded);
}
#[test]
fn postcard_round_trip_timestamp_micros() {
let ts = TimestampMicros::new(1_700_000_000_000_000);
let bytes = postcard::to_allocvec(&ts).unwrap();
let decoded: TimestampMicros = postcard::from_bytes(&bytes).unwrap();
assert_eq!(ts, decoded);
}
#[test]
fn postcard_rejects_invalid_event_type_tag() {
let bytes = postcard::to_allocvec(&0u8).unwrap();
assert!(postcard::from_bytes::<EventTypeTag>(&bytes).is_err());
let bytes = postcard::to_allocvec(&5u8).unwrap();
assert!(postcard::from_bytes::<EventTypeTag>(&bytes).is_err());
let bytes = postcard::to_allocvec(&255u8).unwrap();
assert!(postcard::from_bytes::<EventTypeTag>(&bytes).is_err());
}
#[test]
fn postcard_rejects_oversized_event_length() {
let oversized = MAX_EVENT_PAYLOAD + 1;
let bytes = postcard::to_allocvec(&oversized).unwrap();
assert!(postcard::from_bytes::<EventLength>(&bytes).is_err());
}
#[test]
fn event_sequence_to_sequence_number() {
let es = EventSequence::new(42);
let sn: SequenceNumber = es.into();
assert_eq!(sn.as_i64(), 42);
}
#[test]
fn event_sequence_before_all_to_sequence_number() {
let sn: SequenceNumber = EventSequence::BEFORE_ALL.into();
assert_eq!(sn, SequenceNumber::ZERO);
}
#[test]
fn sequence_number_to_event_sequence() {
let sn = SequenceNumber::from_raw(42);
let es = EventSequence::try_from(sn).unwrap();
assert_eq!(es.raw(), 42);
}
#[test]
fn sequence_number_zero_rejects_to_event_sequence() {
let result = EventSequence::try_from(SequenceNumber::ZERO);
assert!(result.is_err());
}
#[test]
fn sequence_number_negative_rejects_to_event_sequence() {
let result = EventSequence::try_from(SequenceNumber::from_raw(-1));
assert!(result.is_err());
}
#[test]
fn postcard_accepts_max_event_length() {
let bytes = postcard::to_allocvec(&MAX_EVENT_PAYLOAD).unwrap();
let decoded: EventLength = postcard::from_bytes(&bytes).unwrap();
assert_eq!(decoded.raw(), MAX_EVENT_PAYLOAD);
}
}
@@ -0,0 +1,972 @@
use std::io;
use std::sync::Arc;
use tracing::warn;
use crate::io::StorageIO;
use super::manager::SegmentManager;
use super::segment_file::{SEGMENT_HEADER_SIZE, SegmentWriter, ValidEvent};
use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment};
use super::types::{
DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset,
TimestampMicros,
};
#[derive(Debug)]
pub struct SyncResult {
pub synced_through: EventSequence,
pub segment_id: SegmentId,
pub position: SegmentOffset,
pub flushed_events: Vec<ValidEvent>,
}
pub struct EventLogWriter<S: StorageIO> {
manager: Arc<SegmentManager<S>>,
active_writer: SegmentWriter,
active_index: SegmentIndex,
next_seq: EventSequence,
synced_seq: EventSequence,
index_interval: usize,
event_count_in_segment: usize,
last_event_offset: Option<SegmentOffset>,
pending_events: Vec<ValidEvent>,
}
impl<S: StorageIO> EventLogWriter<S> {
pub fn open(manager: Arc<SegmentManager<S>>, index_interval: usize) -> io::Result<Self> {
assert!(index_interval > 0, "index_interval must be positive");
let segments = manager.list_segments()?;
match segments.last() {
None => Self::init_fresh(
manager,
SegmentId::new(1),
EventSequence::new(1),
index_interval,
),
Some(&last_id) => Self::recover_active(manager, &segments, last_id, index_interval),
}
}
fn init_fresh(
manager: Arc<SegmentManager<S>>,
segment_id: SegmentId,
next_seq: EventSequence,
index_interval: usize,
) -> io::Result<Self> {
let fd = manager.open_for_append(segment_id)?;
manager.io().truncate(fd, 0)?;
let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq)?;
writer.sync(manager.io())?;
manager.io().sync_dir(manager.segments_dir())?;
Ok(Self {
manager,
active_writer: writer,
active_index: SegmentIndex::new(),
next_seq,
synced_seq: next_seq.prev_or_before_all(),
index_interval,
event_count_in_segment: 0,
last_event_offset: None,
pending_events: Vec::new(),
})
}
fn recover_active(
manager: Arc<SegmentManager<S>>,
segments: &[SegmentId],
active_id: SegmentId,
index_interval: usize,
) -> io::Result<Self> {
let fd = manager.open_for_append(active_id)?;
let (index, last_seq_in_active) = match rebuild_from_segment(
manager.io(),
fd,
index_interval,
) {
Ok(result) => result,
Err(rebuild_err) => {
let file_size = manager.io().file_size(fd)?;
if file_size <= SEGMENT_HEADER_SIZE as u64 {
manager.io().truncate(fd, 0)?;
let prev_segments = &segments[..segments.len().saturating_sub(1)];
let next_seq = find_last_seq_from_segments(&manager, prev_segments)?
.map_or(EventSequence::new(1), |s| s.next());
return Self::init_fresh(
Arc::clone(&manager),
active_id,
next_seq,
index_interval,
);
}
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"segment {active_id} rebuild failed ({file_size} bytes on disk): {rebuild_err}"
),
));
}
};
let position = SegmentOffset::new(manager.io().file_size(fd)?);
let prev_segments = &segments[..segments.len().saturating_sub(1)];
let next_seq = match last_seq_in_active {
Some(seq) => {
if let Some(sealed_last) = find_last_seq_from_segments(&manager, prev_segments)?
&& seq <= sealed_last
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"active segment last seq ({seq}) must exceed sealed segments' \
last seq ({sealed_last}): cross-segment corruption detected"
),
));
}
seq.next()
}
None => find_last_seq_from_segments(&manager, prev_segments)?
.map_or(EventSequence::new(1), |s| s.next()),
};
let synced_seq = next_seq.prev_or_before_all();
let event_count_in_segment = match (index.first_seq(), index.last_seq()) {
(Some(first), Some(last)) => {
debug_assert!(
first <= last,
"index invariant violated: first_seq {first} > last_seq {last}"
);
usize::try_from(last.raw() - first.raw() + 1).expect("event count exceeds usize")
}
_ => 0,
};
let base_seq = index.first_seq().unwrap_or(next_seq);
let last_event_offset = index.last_seq().and_then(|seq| index.lookup(seq));
let writer = SegmentWriter::resume(
manager.io(),
fd,
active_id,
position,
base_seq,
last_seq_in_active,
);
if let Err(e) = manager.io().delete(&manager.index_path(active_id))
&& e.kind() != io::ErrorKind::NotFound
{
warn!(segment = %active_id, error = %e, "failed to delete stale index");
}
Ok(Self {
manager,
active_writer: writer,
active_index: index,
next_seq,
synced_seq,
index_interval,
event_count_in_segment,
last_event_offset,
pending_events: Vec::new(),
})
}
pub fn append(
&mut self,
did_hash: DidHash,
event_type: EventTypeTag,
payload: Vec<u8>,
) -> io::Result<EventSequence> {
let payload_len = u32::try_from(payload.len())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "payload exceeds u32::MAX"))?;
if payload_len > MAX_EVENT_PAYLOAD {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"payload length {payload_len} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}"
),
));
}
let seq = self.next_seq;
let timestamp = TimestampMicros::now();
let event = ValidEvent {
seq,
timestamp,
did_hash,
event_type,
payload,
};
let offset = self.active_writer.append_event(self.manager.io(), &event)?;
let should_index = self.event_count_in_segment == 0
|| self
.event_count_in_segment
.is_multiple_of(self.index_interval);
if should_index {
self.active_index.record(seq, offset);
}
self.event_count_in_segment = self
.event_count_in_segment
.checked_add(1)
.expect("event_count_in_segment overflow");
self.last_event_offset = Some(offset);
self.next_seq = seq.next();
self.pending_events.push(event);
Ok(seq)
}
pub fn sync(&mut self) -> io::Result<SyncResult> {
if !self.pending_events.is_empty() {
self.active_writer.sync(self.manager.io())?;
}
let flushed = std::mem::take(&mut self.pending_events);
self.synced_seq = flushed.last().map(|e| e.seq).unwrap_or(self.synced_seq);
Ok(SyncResult {
synced_through: self.synced_seq,
segment_id: self.active_writer.segment_id(),
position: self.active_writer.position(),
flushed_events: flushed,
})
}
pub fn rotate_if_needed(&mut self) -> io::Result<Option<SegmentId>> {
if !self.manager.should_rotate(self.active_writer.position()) {
return Ok(None);
}
if !self.pending_events.is_empty() {
return Ok(None);
}
let old_id = self.active_writer.segment_id();
self.ensure_last_event_indexed();
self.manager.seal_segment(old_id, &self.active_index)?;
let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;
match SegmentWriter::new::<S>(self.manager.io(), new_fd, new_id, self.next_seq) {
Ok(writer) => {
self.active_writer = writer;
self.active_index = SegmentIndex::new();
self.event_count_in_segment = 0;
self.last_event_offset = None;
self.manager.commit_rotation(new_id, new_fd);
Ok(Some(old_id))
}
Err(e) => {
self.manager.rollback_rotation(new_id, new_fd);
Err(e)
}
}
}
pub fn checkpoint_index(&self) -> io::Result<()> {
if self.active_index.entry_count() == 0 {
return Ok(());
}
let path = self.manager.index_path(self.active_writer.segment_id());
self.active_index.save(self.manager.io(), &path)
}
pub fn current_seq(&self) -> EventSequence {
self.next_seq.prev_or_before_all()
}
pub fn synced_seq(&self) -> EventSequence {
self.synced_seq
}
pub fn active_segment_id(&self) -> SegmentId {
self.active_writer.segment_id()
}
pub fn active_index_snapshot(&self) -> SegmentIndex {
self.active_index.clone()
}
pub fn position(&self) -> SegmentOffset {
self.active_writer.position()
}
pub fn shutdown(&mut self) -> io::Result<()> {
let _ = self.sync()?;
self.ensure_last_event_indexed();
self.checkpoint_index()
}
fn ensure_last_event_indexed(&mut self) {
let last_written = self.next_seq.prev_or_before_all();
let needs_final_index = self.last_event_offset.is_some()
&& (self.active_index.last_seq() != Some(last_written));
if let (true, Some(offset)) = (needs_final_index, self.last_event_offset) {
self.active_index.record(last_written, offset);
}
}
}
fn find_last_seq_from_segments<S: StorageIO>(
manager: &SegmentManager<S>,
segments: &[SegmentId],
) -> io::Result<Option<EventSequence>> {
segments.iter().rev().try_fold(None, |acc, &seg_id| {
if acc.is_some() {
return Ok(acc);
}
match SegmentIndex::load(manager.io(), &manager.index_path(seg_id)) {
Ok(Some(idx)) => Ok(idx.last_seq()),
Err(e) if e.kind() != io::ErrorKind::InvalidData => Err(e),
_ => {
let fd = manager.open_for_read(seg_id)?;
let (_, last_seq) = rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL)?;
Ok(last_seq)
}
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::eventlog::segment_file::{EVENT_RECORD_OVERHEAD, SegmentReader};
use crate::eventlog::segment_index::DEFAULT_INDEX_INTERVAL;
use crate::sim::SimulatedIO;
use std::path::{Path, PathBuf};
fn setup_manager(max_segment_size: u64) -> Arc<SegmentManager<SimulatedIO>> {
let sim = SimulatedIO::pristine(42);
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap())
}
fn append_test_event(
writer: &mut EventLogWriter<SimulatedIO>,
did_seed: &str,
) -> EventSequence {
writer
.append(
DidHash::from_did(did_seed),
EventTypeTag::COMMIT,
format!("payload-{did_seed}").into_bytes(),
)
.unwrap()
}
#[test]
fn open_fresh_creates_segment() {
let mgr = setup_manager(64 * 1024);
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.active_segment_id(), SegmentId::new(1));
assert_eq!(writer.current_seq(), EventSequence::BEFORE_ALL);
assert_eq!(writer.synced_seq(), EventSequence::BEFORE_ALL);
assert_eq!(
writer.position(),
SegmentOffset::new(SEGMENT_HEADER_SIZE as u64)
);
let segments = mgr.list_segments().unwrap();
assert_eq!(segments, vec![SegmentId::new(1)]);
}
#[test]
fn append_assigns_contiguous_sequences() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let seqs: Vec<EventSequence> = (1..=5)
.map(|i| append_test_event(&mut writer, &format!("did:plc:user{i}")))
.collect();
assert_eq!(seqs, (1..=5).map(EventSequence::new).collect::<Vec<_>>());
assert_eq!(writer.current_seq(), EventSequence::new(5));
}
#[test]
fn sync_returns_flushed_events() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
let result = writer.sync().unwrap();
assert_eq!(result.synced_through, EventSequence::new(3));
assert_eq!(result.flushed_events.len(), 3);
assert_eq!(result.segment_id, SegmentId::new(1));
result
.flushed_events
.iter()
.enumerate()
.for_each(|(i, event)| {
assert_eq!(event.seq, EventSequence::new(i as u64 + 1));
});
assert_eq!(writer.synced_seq(), EventSequence::new(3));
}
#[test]
fn sync_without_pending_is_noop() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let result = writer.sync().unwrap();
assert_eq!(result.synced_through, EventSequence::BEFORE_ALL);
assert!(result.flushed_events.is_empty());
}
#[test]
fn second_sync_returns_only_new_events() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
writer.sync().unwrap();
(4..=5).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
let result = writer.sync().unwrap();
assert_eq!(result.synced_through, EventSequence::new(5));
assert_eq!(result.flushed_events.len(), 2);
assert_eq!(result.flushed_events[0].seq, EventSequence::new(4));
assert_eq!(result.flushed_events[1].seq, EventSequence::new(5));
}
#[test]
fn recovery_preserves_synced_events() {
let mgr = setup_manager(64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=5).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
writer.sync().unwrap();
}
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.current_seq(), EventSequence::new(5));
assert_eq!(writer.synced_seq(), EventSequence::new(5));
assert_eq!(writer.active_segment_id(), SegmentId::new(1));
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(events.len(), 5);
}
#[test]
fn recovery_loses_unsynced_events() {
let mgr = setup_manager(64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
writer.sync().unwrap();
mgr.io().sync_dir(Path::new("/segments")).unwrap();
(4..=6).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
}
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.current_seq(), EventSequence::new(3));
assert_eq!(writer.next_seq, EventSequence::new(4));
}
#[test]
fn rotation_creates_new_segment() {
let payload_size = 100;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 3;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=3).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xAA; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
assert!(writer.rotate_if_needed().unwrap().is_some());
assert_eq!(writer.active_segment_id(), SegmentId::new(2));
assert_eq!(
writer.position(),
SegmentOffset::new(SEGMENT_HEADER_SIZE as u64)
);
let segments = mgr.list_segments().unwrap();
assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(2)]);
}
#[test]
fn rotation_seals_old_segment() {
let payload_size = 100;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=2).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xBB; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
assert!(mgr.is_sealed(SegmentId::new(1)));
let index = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1)))
.unwrap()
.unwrap();
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(2)));
}
#[test]
fn sequences_continue_across_rotation() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=2).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xCC; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
let seq = writer
.append(
DidHash::from_did("did:plc:user3"),
EventTypeTag::COMMIT,
vec![0xCC; payload_size],
)
.unwrap();
assert_eq!(seq, EventSequence::new(3));
}
#[test]
fn recovery_after_rotation() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=2).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xDD; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
writer
.append(
DidHash::from_did("did:plc:user3"),
EventTypeTag::COMMIT,
vec![0xDD; payload_size],
)
.unwrap();
writer.sync().unwrap();
}
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.active_segment_id(), SegmentId::new(2));
assert_eq!(writer.current_seq(), EventSequence::new(3));
assert_eq!(writer.next_seq, EventSequence::new(4));
}
#[test]
fn recovery_sealed_last_segment() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=2).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xEE; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.next_seq, EventSequence::new(3));
}
#[test]
fn recovery_empty_active_after_rotation() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=2).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xEE; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.next_seq, EventSequence::new(3));
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(events.len(), 2);
}
#[test]
fn checkpoint_creates_index_file() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=10).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
writer.sync().unwrap();
writer.checkpoint_index().unwrap();
let wip = mgr.index_path(SegmentId::new(1));
let loaded = SegmentIndex::load(mgr.io(), &wip).unwrap();
assert!(loaded.is_some());
}
#[test]
fn checkpoint_empty_index_is_noop() {
let mgr = setup_manager(64 * 1024);
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
writer.checkpoint_index().unwrap();
let wip = mgr.index_path(SegmentId::new(1));
let loaded = SegmentIndex::load(mgr.io(), &wip).unwrap();
assert!(loaded.is_none());
}
#[test]
fn current_seq_and_synced_seq_diverge_before_sync() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
append_test_event(&mut writer, "did:plc:user1");
append_test_event(&mut writer, "did:plc:user2");
assert_eq!(writer.current_seq(), EventSequence::new(2));
assert_eq!(writer.synced_seq(), EventSequence::BEFORE_ALL);
writer.sync().unwrap();
assert_eq!(writer.current_seq(), EventSequence::new(2));
assert_eq!(writer.synced_seq(), EventSequence::new(2));
}
#[test]
fn sparse_index_built_at_intervals() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 4).unwrap();
(1..=10).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
writer.sync().unwrap();
assert_eq!(writer.active_index.first_seq(), Some(EventSequence::new(1)));
assert!(writer.active_index.entry_count() >= 3);
assert!(writer.active_index.lookup(EventSequence::new(1)).is_some());
assert!(writer.active_index.lookup(EventSequence::new(5)).is_some());
}
#[test]
fn multi_rotation_and_recovery() {
let payload_size = 30;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 3;
let mgr = setup_manager(max_segment_size as u64);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=9).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![i as u8; payload_size],
)
.unwrap();
if i % 3 == 0 {
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
});
writer.sync().unwrap();
}
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.next_seq, EventSequence::new(10));
let segments = mgr.list_segments().unwrap();
assert!(segments.len() >= 3);
}
#[test]
fn shutdown_syncs_and_checkpoints() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=5).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
assert_eq!(writer.synced_seq(), EventSequence::BEFORE_ALL);
writer.shutdown().unwrap();
assert_eq!(writer.synced_seq(), EventSequence::new(5));
let wip = mgr.index_path(SegmentId::new(1));
assert!(SegmentIndex::load(mgr.io(), &wip).unwrap().is_some());
}
#[test]
fn rotation_indexes_last_event() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 5;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=5).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xFF; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
let index = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1)))
.unwrap()
.unwrap();
assert_eq!(index.last_seq(), Some(EventSequence::new(5)));
assert!(index.lookup(EventSequence::new(5)).is_some());
}
#[test]
fn open_idempotent_on_fresh() {
let mgr = setup_manager(64 * 1024);
{
let _writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
}
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.active_segment_id(), SegmentId::new(1));
assert_eq!(writer.current_seq(), EventSequence::BEFORE_ALL);
}
#[test]
fn append_after_recovery_continues_sequence() {
let mgr = setup_manager(64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
writer.sync().unwrap();
}
mgr.shutdown();
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let seq = append_test_event(&mut writer, "did:plc:user4");
assert_eq!(seq, EventSequence::new(4));
writer.sync().unwrap();
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(events.len(), 4);
assert_eq!(events[3].seq, EventSequence::new(4));
}
#[test]
fn recovery_falls_back_to_scan_when_index_corrupt() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
(1..=2).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xAA; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
(3..=4).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:user{i}")),
EventTypeTag::COMMIT,
vec![0xAA; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
mgr.shutdown();
let index_path = mgr.index_path(SegmentId::new(1));
let fd = mgr
.io()
.open(&index_path, crate::OpenOptions::read_write())
.unwrap();
mgr.io().write_all_at(fd, 0, b"CORRUPT_GARBAGE").unwrap();
mgr.io().sync(fd).unwrap();
mgr.io().close(fd).unwrap();
let index_path_2 = mgr.index_path(SegmentId::new(2));
let fd2 = mgr
.io()
.open(&index_path_2, crate::OpenOptions::read_write())
.unwrap();
mgr.io().write_all_at(fd2, 0, b"CORRUPT_GARBAGE").unwrap();
mgr.io().sync(fd2).unwrap();
mgr.io().close(fd2).unwrap();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
assert_eq!(writer.next_seq, EventSequence::new(5));
}
#[test]
fn rotation_not_needed_returns_false() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
append_test_event(&mut writer, "did:plc:user1");
writer.sync().unwrap();
assert!(writer.rotate_if_needed().unwrap().is_none());
}
}
+7
View File
@@ -0,0 +1,7 @@
use std::io;
use crate::blockstore::BlocksSynced;
pub trait PostBlockstoreHook: Send + Sync {
fn on_blocks_synced(&self, proof: &BlocksSynced) -> io::Result<()>;
}
+3 -1
View File
@@ -1,4 +1,6 @@
pub mod blockstore;
pub mod eventlog;
pub mod fsync_order;
mod harness;
mod io;
mod record;
@@ -16,4 +18,4 @@ pub use record::{
FILE_MAGIC, FORMAT_VERSION, HEADER_SIZE, MAX_RECORD_PAYLOAD, RECORD_OVERHEAD, ReadRecord,
RecordReader, RecordWriter,
};
pub use sim::{FaultConfig, SimulatedIO};
pub use sim::{FaultConfig, OpRecord, SimulatedIO};
@@ -0,0 +1,642 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tranquil_store::eventlog::{
DidHash, EVENT_RECORD_OVERHEAD, EventLogWriter, EventSequence, EventTypeTag,
SEGMENT_HEADER_SIZE, SegmentId, SegmentManager, SegmentReader, SegmentWriter, TimestampMicros,
ValidEvent, rebuild_from_segment,
};
use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
fn setup_manager(sim: SimulatedIO, max_segment_size: u64) -> Arc<SegmentManager<SimulatedIO>> {
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap())
}
fn append_test_event(writer: &mut EventLogWriter<SimulatedIO>, seq_hint: u64) -> EventSequence {
writer
.append(
DidHash::from_did(&format!("did:plc:crash{seq_hint}")),
EventTypeTag::COMMIT,
format!("payload-{seq_hint}").into_bytes(),
)
.unwrap()
}
#[test]
fn synced_events_survive_crash() {
(0..500u64).for_each(|seed| {
let sim = SimulatedIO::pristine(seed);
let mgr = setup_manager(sim, 64 * 1024);
let n = 10u64;
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=n).for_each(|i| {
append_test_event(&mut writer, i);
});
writer.sync().unwrap();
mgr.io().sync_dir(Path::new("/segments")).unwrap();
}
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
assert_eq!(
writer.synced_seq(),
EventSequence::new(n),
"seed {seed}: expected all synced events to survive"
);
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(events.len(), n as usize, "seed {seed}");
events.iter().enumerate().for_each(|(i, e)| {
assert_eq!(e.seq, EventSequence::new(i as u64 + 1));
});
});
}
#[test]
fn unsynced_events_lost_on_crash() {
(0..500u64).for_each(|seed| {
let sim = SimulatedIO::pristine(seed);
let mgr = setup_manager(sim, 64 * 1024);
let synced_count = 5u64;
let unsynced_count = 5u64;
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=synced_count).for_each(|i| {
append_test_event(&mut writer, i);
});
writer.sync().unwrap();
mgr.io().sync_dir(Path::new("/segments")).unwrap();
(synced_count + 1..=synced_count + unsynced_count).for_each(|i| {
append_test_event(&mut writer, i);
});
}
mgr.shutdown();
mgr.io().crash();
let writer =
EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
let recovered_count = writer.synced_seq().raw();
assert_eq!(
recovered_count, synced_count,
"seed {seed}: pristine IO should recover exactly {synced_count} synced events, got {recovered_count}"
);
});
}
#[test]
fn sequence_monotonicity_after_recovery() {
(0..500u64).for_each(|seed| {
let sim = SimulatedIO::new(seed, FaultConfig::moderate());
let mgr = setup_manager(sim, 64 * 1024);
let crash_point = (seed % 15) + 3;
let write_result: Result<(), std::io::Error> = (|| {
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256)?;
(1..=crash_point).try_for_each(|i| -> std::io::Result<()> {
writer.append(
DidHash::from_did(&format!("did:plc:mono{i}")),
EventTypeTag::COMMIT,
format!("data-{i}").into_bytes(),
)?;
if i % 3 == 0 {
writer.sync()?;
mgr.io().sync_dir(Path::new("/segments"))?;
}
Ok(())
})?;
Ok(())
})();
let _ = write_result;
mgr.shutdown();
mgr.io().crash();
let mgr_clone = Arc::clone(&mgr);
let recovery_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut writer = EventLogWriter::open(Arc::clone(&mgr_clone), 256)?;
let new_seqs: Vec<EventSequence> = (0..5u64)
.filter_map(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:post{i}")),
EventTypeTag::COMMIT,
format!("post-recovery-{i}").into_bytes(),
)
.ok()
})
.collect();
Ok::<_, std::io::Error>(new_seqs)
}));
let Ok(Ok(new_seqs)) = recovery_result else {
return;
};
new_seqs.windows(2).for_each(|pair| {
assert!(
pair[1].raw() == pair[0].raw() + 1,
"seed {seed}: non-contiguous seqs {} -> {}",
pair[0],
pair[1],
);
});
if let Some(first_new) = new_seqs.first() {
assert!(first_new.raw() > 0, "seed {seed}: new sequence starts at 0");
}
});
}
#[test]
fn partial_event_truncated_on_recovery() {
(0..500u64).for_each(|seed| {
let sim = SimulatedIO::pristine(seed);
let mgr = setup_manager(sim, 64 * 1024);
let complete_count = 5u64;
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=complete_count).for_each(|i| {
append_test_event(&mut writer, i);
});
writer.sync().unwrap();
mgr.io().sync_dir(Path::new("/segments")).unwrap();
}
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let file_size = mgr.io().file_size(fd).unwrap();
let partial_bytes = ((seed % 20) + 1) as usize;
let junk: Vec<u8> = (0..partial_bytes)
.map(|i| (i as u8).wrapping_add(seed as u8))
.collect();
mgr.io().write_all_at(fd, file_size, &junk).unwrap();
mgr.io().sync(fd).unwrap();
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
assert_eq!(
writer.synced_seq(),
EventSequence::new(complete_count),
"seed {seed}: partial write should be truncated, preserving {complete_count} events"
);
});
}
#[test]
fn cross_segment_recovery() {
(0..200u64).for_each(|seed| {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let events_per_segment = 3;
let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64;
let sim = SimulatedIO::pristine(seed);
let mgr = setup_manager(sim, max_segment_size);
let sealed_events = 9u64;
let trailing_unsynced = 2u64;
let total_events = sealed_events + trailing_unsynced;
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=total_events).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:xseg{i}")),
EventTypeTag::COMMIT,
vec![i as u8; payload_size],
)
.unwrap();
if i % events_per_segment as u64 == 0 && i <= sealed_events {
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
});
mgr.io().sync_dir(Path::new("/segments")).unwrap();
}
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
let recovered = writer.synced_seq().raw();
let sealed_segments = mgr.list_segments().unwrap();
let sealed_count = sealed_segments.len().saturating_sub(1);
assert!(
recovered >= (sealed_count as u64) * events_per_segment as u64,
"seed {seed}: recovered {recovered} but expected at least {} sealed events",
sealed_count * events_per_segment,
);
sealed_segments[..sealed_count].iter().for_each(|&seg_id| {
let fd = mgr.open_for_read(seg_id).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
.unwrap()
.valid_prefix()
.unwrap();
assert_eq!(
events.len(),
events_per_segment,
"seed {seed}: sealed segment {seg_id} should have {events_per_segment} events"
);
});
});
}
#[test]
fn corrupt_index_triggers_rebuild() {
(0..200u64).for_each(|seed| {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * 3) as u64;
let sim = SimulatedIO::pristine(seed);
let mgr = setup_manager(sim, max_segment_size);
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=6).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:idx{i}")),
EventTypeTag::COMMIT,
vec![0xAA; payload_size],
)
.unwrap();
if i % 3 == 0 {
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
});
writer.sync().unwrap();
}
mgr.shutdown();
let index_path = mgr.index_path(SegmentId::new(1));
if let Ok(fd) = mgr.io().open(&index_path, OpenOptions::read_write()) {
mgr.io()
.write_all_at(fd, 0, b"CORRUPT_INDEX_GARBAGE_DATA_XYZ")
.unwrap();
mgr.io().sync(fd).unwrap();
mgr.io().close(fd).unwrap();
}
let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
assert!(
writer.synced_seq().raw() >= 6,
"seed {seed}: recovery after corrupt index should find all events, got seq {}",
writer.synced_seq(),
);
});
}
#[test]
fn large_sealed_segment_index_rebuild_latency() {
let payload_size = 1024;
let event_count = 64_000u64;
let sim = SimulatedIO::pristine(42);
let mgr = setup_manager(sim, 256 * 1024 * 1024);
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=event_count).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:bench{i}")),
EventTypeTag::COMMIT,
vec![0xBB; payload_size],
)
.unwrap();
});
writer.sync().unwrap();
writer.checkpoint_index().unwrap();
}
mgr.shutdown();
let index_path = mgr.index_path(SegmentId::new(1));
let _ = mgr.io().delete(&index_path);
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let start = std::time::Instant::now();
let (index, last_seq) = rebuild_from_segment(mgr.io(), fd, 256).unwrap();
let elapsed = start.elapsed();
assert_eq!(last_seq, Some(EventSequence::new(event_count)));
assert!(index.entry_count() > 0);
assert!(
elapsed.as_secs() < 2,
"index rebuild took {:?}, exceeds 2s budget",
elapsed,
);
}
#[test]
fn corrupt_metadata_triggers_scan() {
let sim = SimulatedIO::pristine(42);
let mgr = setup_manager(sim, 64 * 1024);
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=10).for_each(|i| {
append_test_event(&mut writer, i);
});
writer.sync().unwrap();
writer.checkpoint_index().unwrap();
}
mgr.shutdown();
let index_path = mgr.index_path(SegmentId::new(1));
if let Ok(fd) = mgr.io().open(&index_path, OpenOptions::read_write()) {
mgr.io()
.write_all_at(fd, 0, b"TOTALLY_CORRUPT_META")
.unwrap();
mgr.io().sync(fd).unwrap();
mgr.io().close(fd).unwrap();
}
let writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
assert_eq!(
writer.synced_seq(),
EventSequence::new(10),
"recovery via segment scan should find all 10 events"
);
}
#[test]
fn pristine_comparison_under_faults() {
(0..500u64).for_each(|seed| {
let event_count = 15u64;
let sync_interval = 5u64;
let pristine_sim = SimulatedIO::pristine(seed);
let pristine_mgr = setup_manager(pristine_sim, 64 * 1024);
{
let mut writer = EventLogWriter::open(Arc::clone(&pristine_mgr), 256).unwrap();
(1..=event_count).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:prist{i}")),
EventTypeTag::COMMIT,
format!("pristine-{i}").into_bytes(),
)
.unwrap();
if i % sync_interval == 0 {
writer.sync().unwrap();
}
});
writer.sync().unwrap();
}
pristine_mgr.shutdown();
let pristine_fd = pristine_mgr.open_for_read(SegmentId::new(1)).unwrap();
let pristine_events = SegmentReader::open(pristine_mgr.io(), pristine_fd)
.unwrap()
.valid_prefix()
.unwrap();
let faulty_sim = SimulatedIO::new(seed, FaultConfig::moderate());
let faulty_mgr = setup_manager(faulty_sim, 64 * 1024);
let write_ok = (|| -> std::io::Result<()> {
let mut writer = EventLogWriter::open(Arc::clone(&faulty_mgr), 256)?;
(1..=event_count).try_for_each(|i| -> std::io::Result<()> {
writer.append(
DidHash::from_did(&format!("did:plc:prist{i}")),
EventTypeTag::COMMIT,
format!("pristine-{i}").into_bytes(),
)?;
if i % sync_interval == 0 {
let _ = writer.sync();
let _ = faulty_mgr.io().sync_dir(Path::new("/segments"));
}
Ok(())
})?;
let _ = writer.sync();
Ok(())
})();
let _ = write_ok;
faulty_mgr.shutdown();
faulty_mgr.io().crash();
let faulty_clone = Arc::clone(&faulty_mgr);
let recovery = std::panic::catch_unwind(std::panic::AssertUnwindSafe(
|| -> std::io::Result<Option<Vec<ValidEvent>>> {
let recovered_writer = EventLogWriter::open(Arc::clone(&faulty_clone), 256)?;
let recovered_seq = recovered_writer.synced_seq().raw();
assert!(
recovered_seq <= event_count,
"seed {seed}: recovered {recovered_seq} > written {event_count}"
);
if recovered_seq == 0 {
return Ok(None);
}
let fd = faulty_clone.open_for_read(SegmentId::new(1))?;
let events = SegmentReader::open(faulty_clone.io(), fd)?.valid_prefix()?;
Ok(Some(events))
},
));
if let Ok(Ok(Some(recovered_events))) = recovery {
let is_prefix = recovered_events
.iter()
.zip(pristine_events.iter())
.all(|(r, p)| r.seq == p.seq && r.payload == p.payload);
assert!(
is_prefix,
"seed {seed}: recovered events must be a prefix of pristine"
);
}
});
}
#[test]
fn bit_flip_detected_by_checksum() {
(0..1000u64).for_each(|seed| {
let sim = SimulatedIO::pristine(seed);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read_write())
.unwrap();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let data_len = ((seed % 256) as usize).max(1);
let event = ValidEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(1_000_000),
did_hash: DidHash::from_did("did:plc:bitflip"),
event_type: EventTypeTag::COMMIT,
payload: vec![0xAA; data_len],
};
writer.append_event(&sim, &event).unwrap();
writer.sync(&sim).unwrap();
let record_start = SEGMENT_HEADER_SIZE as u64;
let record_end = record_start + EVENT_RECORD_OVERHEAD as u64 + data_len as u64;
let flip_pos = record_start + (seed.wrapping_mul(7) % (record_end - record_start));
let flip_bit = (seed.wrapping_mul(13) % 8) as u8;
let mut byte_buf = [0u8; 1];
sim.read_exact_at(fd, flip_pos, &mut byte_buf).unwrap();
byte_buf[0] ^= 1 << flip_bit;
sim.write_all_at(fd, flip_pos, &byte_buf).unwrap();
use tranquil_store::eventlog::ReadEventRecord;
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(
!matches!(record, ReadEventRecord::Valid { .. }),
"seed {seed}: bit flip at offset {flip_pos} bit {flip_bit} was not detected"
);
});
}
fn fault_configs() -> Vec<(&'static str, FaultConfig)> {
vec![
(
"partial_writes_only",
FaultConfig {
partial_write_probability: 0.15,
..FaultConfig::none()
},
),
(
"sync_failures_only",
FaultConfig {
sync_failure_probability: 0.10,
dir_sync_failure_probability: 0.05,
..FaultConfig::none()
},
),
("combined", FaultConfig::moderate()),
(
"bit_flips_only",
FaultConfig {
bit_flip_on_read_probability: 0.05,
..FaultConfig::none()
},
),
]
}
#[test]
fn pristine_comparison_parameterized_faults() {
fault_configs().iter().for_each(|(config_name, config)| {
(0..200u64).for_each(|seed| {
let event_count = 10u64;
let pristine_sim = SimulatedIO::pristine(seed);
let pristine_mgr = setup_manager(pristine_sim, 64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&pristine_mgr), 256).unwrap();
(1..=event_count).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:param{i}")),
EventTypeTag::COMMIT,
format!("param-{i}").into_bytes(),
)
.unwrap();
if i % 4 == 0 {
writer.sync().unwrap();
}
});
writer.sync().unwrap();
}
pristine_mgr.shutdown();
let pristine_fd = pristine_mgr.open_for_read(SegmentId::new(1)).unwrap();
let pristine_events = SegmentReader::open(pristine_mgr.io(), pristine_fd)
.unwrap()
.valid_prefix()
.unwrap();
let faulty_sim = SimulatedIO::new(seed, *config);
let faulty_mgr = setup_manager(faulty_sim, 64 * 1024);
let _ = (|| -> std::io::Result<()> {
let mut writer =
EventLogWriter::open(Arc::clone(&faulty_mgr), 256)?;
(1..=event_count).try_for_each(|i| -> std::io::Result<()> {
writer.append(
DidHash::from_did(&format!("did:plc:param{i}")),
EventTypeTag::COMMIT,
format!("param-{i}").into_bytes(),
)?;
if i % 4 == 0 {
let _ = writer.sync();
let _ = faulty_mgr.io().sync_dir(Path::new("/segments"));
}
Ok(())
})?;
let _ = writer.sync();
Ok(())
})();
faulty_mgr.shutdown();
faulty_mgr.io().crash();
let faulty_clone = Arc::clone(&faulty_mgr);
let recovery = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> std::io::Result<Option<Vec<ValidEvent>>> {
let recovered_writer =
EventLogWriter::open(Arc::clone(&faulty_clone), 256)?;
let recovered_seq = recovered_writer.synced_seq().raw();
assert!(
recovered_seq <= event_count,
"config={config_name} seed={seed}: recovered {recovered_seq} > written {event_count}"
);
if recovered_seq == 0 {
return Ok(None);
}
let fd = faulty_clone.open_for_read(SegmentId::new(1))?;
let events = SegmentReader::open(faulty_clone.io(), fd)?
.valid_prefix()?;
Ok(Some(events))
}));
if let Ok(Ok(Some(recovered_events))) = recovery {
let is_prefix = recovered_events
.iter()
.zip(pristine_events.iter())
.all(|(r, p)| r.seq == p.seq && r.payload == p.payload);
assert!(
is_prefix,
"config={config_name} seed={seed}: recovered is not prefix of pristine"
);
}
});
});
}
@@ -0,0 +1,677 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tranquil_store::eventlog::{
DidHash, EVENT_RECORD_OVERHEAD, EventLog, EventLogConfig, EventLogReader, EventLogWriter,
EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, PayloadError, RawEvent, SEGMENT_HEADER_SIZE,
SegmentId, SegmentIndex, SegmentManager, SegmentReader, TimestampMicros, ValidEvent,
decode_payload, encode_payload, to_sequenced_event, validate_payload_size,
};
use tranquil_store::{OpRecord, OpenOptions, SimulatedIO, StorageIO};
fn setup_manager(max_segment_size: u64) -> Arc<SegmentManager<SimulatedIO>> {
let sim = SimulatedIO::pristine(42);
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap())
}
fn append_test_event(writer: &mut EventLogWriter<SimulatedIO>, seq_hint: u64) -> EventSequence {
writer
.append(
DidHash::from_did(&format!("did:plc:prop{seq_hint}")),
EventTypeTag::COMMIT,
format!("payload-{seq_hint}").into_bytes(),
)
.unwrap()
}
#[test]
fn sequence_assignment_is_contiguous() {
let n = 100u64;
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
let seqs: Vec<EventSequence> = (1..=n).map(|i| append_test_event(&mut writer, i)).collect();
seqs.iter().enumerate().for_each(|(i, seq)| {
assert_eq!(
seq.raw(),
i as u64 + 1,
"event {i} should have seq {}",
i + 1,
);
});
}
#[test]
fn cursor_resumption_returns_correct_suffix() {
let mgr = setup_manager(64 * 1024);
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=1000).for_each(|i| {
append_test_event(&mut writer, i);
});
writer.shutdown().unwrap();
}
mgr.shutdown();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
reader.refresh_segment_ranges().unwrap();
let events = reader
.read_events_from(EventSequence::new(500), 1000)
.unwrap();
assert_eq!(events.len(), 500);
assert_eq!(events[0].seq, EventSequence::new(501));
assert_eq!(events[499].seq, EventSequence::new(1000));
events.windows(2).for_each(|pair| {
assert_eq!(
pair[1].seq.raw(),
pair[0].seq.raw() + 1,
"gap between {} and {}",
pair[0].seq,
pair[1].seq,
);
});
}
#[test]
fn cross_segment_read_is_seamless() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let events_per_segment = 10;
let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64;
let total_events = 100u64;
let mgr = setup_manager(max_segment_size);
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=total_events).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:xseg{i}")),
EventTypeTag::COMMIT,
vec![i as u8; payload_size],
)
.unwrap();
if i % events_per_segment as u64 == 0 && i < total_events {
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
});
writer.shutdown().unwrap();
}
mgr.shutdown();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
reader.refresh_segment_ranges().unwrap();
let events = reader
.read_events_from(EventSequence::BEFORE_ALL, total_events as usize + 10)
.unwrap();
assert_eq!(events.len(), total_events as usize);
events.iter().enumerate().for_each(|(i, e)| {
assert_eq!(
e.seq,
EventSequence::new(i as u64 + 1),
"event at index {i} has wrong seq"
);
});
let mut seen = std::collections::HashSet::new();
events.iter().for_each(|e| {
assert!(seen.insert(e.seq.raw()), "duplicate seq {}", e.seq,);
});
}
#[test]
fn retention_deletes_only_old_segments() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let events_per_segment = 3;
let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64;
let sim = SimulatedIO::pristine(42);
let mgr =
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap());
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=15).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:ret{i}")),
EventTypeTag::COMMIT,
vec![0xAA; payload_size],
)
.unwrap();
if i % events_per_segment as u64 == 0 {
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
});
writer.sync().unwrap();
let segments_before = mgr.list_segments().unwrap();
assert!(segments_before.len() >= 5);
let segments_to_delete: Vec<_> = segments_before[..2].to_vec();
segments_to_delete.iter().for_each(|&id| {
mgr.delete_segment(id).unwrap();
});
let segments_after = mgr.list_segments().unwrap();
assert_eq!(segments_after.len(), segments_before.len() - 2,);
segments_to_delete.iter().for_each(|id| {
assert!(
!segments_after.contains(id),
"deleted segment {id} still present"
);
});
segments_after.iter().for_each(|id| {
assert!(
!segments_to_delete.contains(id),
"remaining segment {id} was supposed to be deleted"
);
});
}
#[test]
fn did_hash_is_deterministic() {
let dids = [
"did:plc:abc123",
"did:plc:xyz789",
"did:web:example.com",
"did:plc:aaaabbbbccccddddeeeeffffggg",
];
dids.iter().for_each(|did| {
let h1 = DidHash::from_did(did);
let h2 = DidHash::from_did(did);
assert_eq!(h1, h2, "DidHash not deterministic for {did}");
});
}
#[test]
fn payload_round_trip() {
use bytes::Bytes;
use tranquil_db_traits::{AccountStatus, RepoEventType, SequenceNumber, SequencedEvent};
use tranquil_types::{Did, Handle};
let variants: Vec<(RepoEventType, EventTypeTag, SequencedEvent)> = vec![
(
RepoEventType::Commit,
EventTypeTag::COMMIT,
SequencedEvent {
seq: SequenceNumber::from_raw(1),
did: Did::new("did:plc:testuser1234567890abcdef").unwrap(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Commit,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: Some(
serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}]),
),
blobs: Some(vec!["bafkreibtest".to_owned()]),
blocks_cids: None,
handle: None,
active: None,
status: None,
rev: Some("rev1".to_owned()),
},
),
(
RepoEventType::Identity,
EventTypeTag::IDENTITY,
SequencedEvent {
seq: SequenceNumber::from_raw(2),
did: Did::new("did:plc:testuser1234567890abcdef").unwrap(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Identity,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: Some(Handle::new("test.bsky.social").unwrap()),
active: None,
status: None,
rev: None,
},
),
(
RepoEventType::Account,
EventTypeTag::ACCOUNT,
SequencedEvent {
seq: SequenceNumber::from_raw(3),
did: Did::new("did:plc:testuser1234567890abcdef").unwrap(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Account,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: None,
active: Some(true),
status: Some(AccountStatus::Active),
rev: None,
},
),
(
RepoEventType::Sync,
EventTypeTag::SYNC,
SequencedEvent {
seq: SequenceNumber::from_raw(4),
did: Did::new("did:plc:testuser1234567890abcdef").unwrap(),
created_at: chrono::Utc::now(),
event_type: RepoEventType::Sync,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: None,
active: None,
status: None,
rev: None,
},
),
];
variants.iter().for_each(|(event_type, tag, event)| {
let encoded = encode_payload(event);
let decoded = decode_payload(&encoded).unwrap();
let raw = RawEvent {
seq: EventSequence::new(event.seq.as_i64() as u64),
timestamp: TimestampMicros::now(),
did_hash: DidHash::from_did(event.did.as_str()),
event_type: *tag,
payload: Bytes::from(encoded),
};
let reconstructed = to_sequenced_event(&raw, &decoded).unwrap();
assert_eq!(reconstructed.did.as_str(), event.did.as_str());
assert_eq!(reconstructed.event_type, *event_type);
assert_eq!(reconstructed.rev, event.rev);
assert_eq!(reconstructed.blobs, event.blobs);
assert_eq!(reconstructed.active, event.active);
});
}
#[test]
fn max_payload_accepted() {
let payload = vec![0xBB; MAX_EVENT_PAYLOAD as usize];
assert!(validate_payload_size(&payload).is_ok());
let sim = SimulatedIO::pristine(42);
let dir = Path::new("/test");
sim.mkdir(dir).unwrap();
sim.sync_dir(dir).unwrap();
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read_write())
.unwrap();
let mut writer = tranquil_store::eventlog::SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
)
.unwrap();
let event = ValidEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(1_000_000),
did_hash: DidHash::from_did("did:plc:maxpayload"),
event_type: EventTypeTag::COMMIT,
payload: payload.clone(),
};
writer.append_event(&sim, &event).unwrap();
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].payload.len(), MAX_EVENT_PAYLOAD as usize);
}
#[test]
fn oversized_payload_rejected() {
let payload = vec![0xCC; MAX_EVENT_PAYLOAD as usize + 1];
match validate_payload_size(&payload) {
Err(PayloadError::TooLarge { size, max }) => {
assert_eq!(size, MAX_EVENT_PAYLOAD as usize + 1);
assert_eq!(max, MAX_EVENT_PAYLOAD as usize);
}
other => panic!("expected TooLarge, got {other:?}"),
}
}
#[test]
fn retention_does_not_break_active_readers() {
let payload_size = 50;
let record_size = EVENT_RECORD_OVERHEAD + payload_size;
let events_per_segment = 5;
let max_segment_size = (SEGMENT_HEADER_SIZE + record_size * events_per_segment) as u64;
let sim = SimulatedIO::pristine(42);
let mgr =
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap());
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=25).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:active{i}")),
EventTypeTag::COMMIT,
vec![i as u8; payload_size],
)
.unwrap();
if i % events_per_segment as u64 == 0 {
writer.sync().unwrap();
writer.rotate_if_needed().unwrap();
}
});
writer.sync().unwrap();
}
mgr.shutdown();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
reader.refresh_segment_ranges().unwrap();
let first_batch = reader
.read_events_from(EventSequence::BEFORE_ALL, 10)
.unwrap();
assert_eq!(first_batch.len(), 10);
mgr.delete_segment(SegmentId::new(1)).unwrap();
reader.invalidate_index(SegmentId::new(1));
reader.invalidate_mmap(SegmentId::new(1));
reader.refresh_segment_ranges().unwrap();
let later_events = reader.read_events_from(EventSequence::new(10), 20).unwrap();
assert!(!later_events.is_empty());
later_events.iter().for_each(|e| {
assert!(e.seq.raw() > 10);
});
}
#[tokio::test]
async fn subscriber_lag_recovery() {
let sim = SimulatedIO::pristine(42);
let config = EventLogConfig {
segments_dir: PathBuf::from("/segments"),
max_segment_size: 64 * 1024,
index_interval: 256,
broadcast_buffer: 4,
use_mmap: false,
};
let event_log = EventLog::open(config, sim).unwrap();
let mut subscriber = event_log.subscriber(EventSequence::BEFORE_ALL);
let total_events = 20u64;
(1..=total_events).for_each(|i| {
event_log
.append_and_sync(
&tranquil_types::Did::new("did:plc:testuser1234567890abcdef").unwrap(),
tranquil_db_traits::RepoEventType::Commit,
&tranquil_db_traits::SequencedEvent {
seq: tranquil_db_traits::SequenceNumber::from_raw(i as i64),
did: tranquil_types::Did::new("did:plc:testuser1234567890abcdef").unwrap(),
created_at: chrono::Utc::now(),
event_type: tranquil_db_traits::RepoEventType::Commit,
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
handle: None,
active: None,
status: None,
rev: None,
},
)
.unwrap();
});
let mut received_seqs: Vec<u64> = Vec::new();
let timeout = tokio::time::timeout(Duration::from_secs(5), async {
while let Some(event) = subscriber.next().await {
received_seqs.push(event.seq.raw());
if event.seq.raw() >= total_events {
break;
}
}
});
timeout
.await
.expect("subscriber timed out before receiving all events");
assert_eq!(
received_seqs.len(),
total_events as usize,
"subscriber should receive all {total_events} events, got {}",
received_seqs.len(),
);
received_seqs.windows(2).for_each(|pair| {
assert!(
pair[1] > pair[0],
"events must be in order: {} -> {}",
pair[0],
pair[1],
);
});
let unique: std::collections::HashSet<u64> = received_seqs.iter().copied().collect();
assert_eq!(
unique.len(),
received_seqs.len(),
"no duplicate events allowed"
);
}
#[test]
fn index_checkpoint_accelerates_recovery() {
let event_count = 50_000u64;
let sim = SimulatedIO::pristine(42);
let mgr =
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), 256 * 1024 * 1024).unwrap());
{
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
(1..=event_count).for_each(|i| {
writer
.append(
DidHash::from_did(&format!("did:plc:chk{i}")),
EventTypeTag::COMMIT,
format!("ckpt-{i}").into_bytes(),
)
.unwrap();
});
writer.shutdown().unwrap();
}
mgr.shutdown();
let index = SegmentIndex::load(mgr.io(), &mgr.index_path(SegmentId::new(1)))
.unwrap()
.unwrap();
assert!(index.entry_count() > 0);
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(event_count)));
let mid = EventSequence::new(event_count / 2);
let offset = index.lookup(mid);
assert!(offset.is_some(), "index should cover midpoint seq {}", mid,);
let reader_with_index = EventLogReader::new(Arc::clone(&mgr), false);
let reads_before = mgr
.io()
.op_log()
.iter()
.filter(|op| matches!(op, OpRecord::ReadAt { .. }))
.count();
reader_with_index.refresh_segment_ranges().unwrap();
let mid_events = reader_with_index
.read_events_from(EventSequence::new(event_count / 2), 10)
.unwrap();
assert_eq!(mid_events.len(), 10);
let reads_with_index = mgr
.io()
.op_log()
.iter()
.filter(|op| matches!(op, OpRecord::ReadAt { .. }))
.count()
- reads_before;
let _ = mgr.io().delete(&mgr.index_path(SegmentId::new(1)));
let reader_without_index = EventLogReader::new(Arc::clone(&mgr), false);
let reads_before = mgr
.io()
.op_log()
.iter()
.filter(|op| matches!(op, OpRecord::ReadAt { .. }))
.count();
reader_without_index.refresh_segment_ranges().unwrap();
let mid_events_no_idx = reader_without_index
.read_events_from(EventSequence::new(event_count / 2), 10)
.unwrap();
assert_eq!(mid_events_no_idx.len(), 10);
let reads_without_index = mgr
.io()
.op_log()
.iter()
.filter(|op| matches!(op, OpRecord::ReadAt { .. }))
.count()
- reads_before;
assert!(
reads_with_index < reads_without_index,
"read with index ({reads_with_index} reads) should require fewer reads than without ({reads_without_index} reads)"
);
}
#[test]
fn fsync_ordering_blocks_before_events() {
use tranquil_store::blockstore::{
CID_SIZE, DataFileId, DataFileManager, DataFileReader, DataFileWriter,
};
fn test_cid(seed: u8) -> [u8; CID_SIZE] {
let mut cid = [0u8; CID_SIZE];
cid[0] = 0x01;
cid[1] = 0x71;
cid[2] = 0x12;
cid[3] = 0x20;
cid[4] = seed;
cid
}
let sim = Arc::new(SimulatedIO::pristine(42));
let data_dir = Path::new("/blocks");
sim.mkdir(data_dir).unwrap();
sim.sync_dir(data_dir).unwrap();
let seg_dir = Path::new("/segments");
let block_mgr =
DataFileManager::with_default_max_size(Arc::clone(&sim), data_dir.to_path_buf());
let event_mgr = Arc::new(
SegmentManager::new(Arc::clone(&sim), PathBuf::from("/segments"), 64 * 1024).unwrap(),
);
let block_fd = block_mgr.open_for_append(DataFileId::new(0)).unwrap();
let mut block_writer =
DataFileWriter::new(block_mgr.io(), block_fd, DataFileId::new(0)).unwrap();
let cid = test_cid(1);
let _ = block_writer.append_block(&cid, &[0xAA; 128]).unwrap();
block_writer.sync().unwrap();
sim.sync_dir(data_dir).unwrap();
{
let mut event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap();
event_writer
.append(
DidHash::from_did("did:plc:fsyncorder"),
EventTypeTag::COMMIT,
b"event-before-sync".to_vec(),
)
.unwrap();
}
sim.crash();
event_mgr.shutdown();
let block_fd = sim
.open(
Path::new("/blocks/000000.tqb"),
OpenOptions::read_only_existing(),
)
.unwrap();
let block_reader = DataFileReader::open(&*sim, block_fd).unwrap();
let recovered_blocks = block_reader.valid_blocks().unwrap();
assert_eq!(
recovered_blocks.len(),
1,
"blockstore was synced, block must survive crash"
);
assert_eq!(recovered_blocks[0].1, cid, "recovered block CID must match");
let event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap();
assert_eq!(
event_writer.synced_seq(),
EventSequence::BEFORE_ALL,
"crash between blockstore sync and eventlog sync must not persist the event (blocks exist, event does not = orphan, not inconsistency)"
);
drop(event_writer);
{
let mut event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap();
event_writer
.append(
DidHash::from_did("did:plc:fsyncorder"),
EventTypeTag::COMMIT,
b"event-with-sync".to_vec(),
)
.unwrap();
event_writer.sync().unwrap();
sim.sync_dir(seg_dir).unwrap();
}
event_mgr.shutdown();
sim.crash();
let event_writer = EventLogWriter::open(Arc::clone(&event_mgr), 256).unwrap();
assert_eq!(
event_writer.synced_seq(),
EventSequence::new(1),
"both stores synced, event must survive crash"
);
}