feat(tranquil-store): cargo-fuzz targets with asan+ubsan

Lewis: May this revision serve well! <lu5a@proton.me>

crates/tranquil-store/fuzz/.gitignore

@@ -0,0 +1,4 @@
+target/
+artifacts/
+coverage/
+Cargo.lock

crates/tranquil-store/fuzz/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "tranquil-store-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2024"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+arbitrary = { version = "1", features = ["derive"] }
+tranquil-store = { path = "..", features = ["test-harness"] }
+tempfile = "3"
+tokio = { version = "1", features = ["rt", "time", "macros", "sync"] }
+
+[[bin]]
+name = "decode_block_record"
+path = "fuzz_targets/decode_block_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "decode_hint_record"
+path = "fuzz_targets/decode_hint_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "segment_scan"
+path = "fuzz_targets/segment_scan.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "metastore_key_codec"
+path = "fuzz_targets/metastore_key_codec.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "gauntlet_micro"
+path = "fuzz_targets/gauntlet_micro.rs"
+test = false
+doc = false
+bench = false
+
+[workspace]

crates/tranquil-store/fuzz/corpus/decode_block_record/cid_only

@@ -0,0 +1 @@
+

crates/tranquil-store/fuzz/corpus/segment_scan/bad_magic

@@ -0,0 +1 @@
+BADX

crates/tranquil-store/fuzz/corpus/segment_scan/header_only

@@ -0,0 +1 @@
+TQEV

crates/tranquil-store/fuzz/fuzz_targets/decode_block_record.rs

@@ -0,0 +1,28 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{BlockOffset, decode_block_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/block.tqb"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let _ = decode_block_record(&sim, fd, BlockOffset::new(0), file_size);
+    let _ = sim.close(fd);
+});

crates/tranquil-store/fuzz/fuzz_targets/decode_hint_record.rs

@@ -0,0 +1,41 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{HintOffset, decode_hint_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/hint.tqh"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let cursor = std::cell::Cell::new(0u64);
+    std::iter::from_fn(|| {
+        if cursor.get() >= file_size {
+            return None;
+        }
+        match decode_hint_record(&sim, fd, HintOffset::new(cursor.get()), file_size) {
+            Ok(Some(_)) => {
+                cursor.set(cursor.get() + 64);
+                Some(())
+            }
+            _ => None,
+        }
+    })
+    .for_each(|()| {});
+    let _ = sim.close(fd);
+});

crates/tranquil-store/fuzz/fuzz_targets/gauntlet_micro.rs

@@ -0,0 +1,109 @@
+#![no_main]
+
+use std::sync::OnceLock;
+
+use arbitrary::{Arbitrary, Unstructured};
+use libfuzzer_sys::fuzz_target;
+use tokio::runtime::Runtime;
+use tranquil_store::blockstore::GroupCommitConfig;
+use tranquil_store::gauntlet::{
+    CollectionName, DidSpaceSize, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize,
+    MaxFileSize, Op, OpCount, OpInterval, OpStream, OpWeights, RecordKey, RestartPolicy,
+    RetentionMaxSecs, RunLimits, Seed, ShardCount, SizeDistribution, StoreConfig, ValueBytes,
+    ValueSeed, WallMs, WorkloadModel, WriterConcurrency,
+};
+
+#[derive(Arbitrary, Debug)]
+enum FuzzOp {
+    Add { rkey: u8, value: u16 },
+    Delete { rkey: u8 },
+    Compact,
+    Checkpoint,
+    Read { rkey: u8 },
+    ReadBlock { value: u16 },
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+const MAX_OPS: usize = 128;
+
+fn to_op(fuzz_op: FuzzOp) -> Op {
+    match fuzz_op {
+        FuzzOp::Add { rkey, value } => Op::AddRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+            value_seed: ValueSeed(u32::from(value)),
+        },
+        FuzzOp::Delete { rkey } => Op::DeleteRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::Compact => Op::Compact,
+        FuzzOp::Checkpoint => Op::Checkpoint,
+        FuzzOp::Read { rkey } => Op::ReadRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::ReadBlock { value } => Op::ReadBlock {
+            value_seed: ValueSeed(u32::from(value)),
+        },
+    }
+}
+
+fn tiny_config() -> GauntletConfig {
+    GauntletConfig {
+        seed: Seed(0),
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights::default(),
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: vec![CollectionName(COLLECTION.to_string())],
+            key_space: KeySpaceSize(256),
+            did_space: DidSpaceSize(8),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(0),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::READ_AFTER_WRITE,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(2_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(32)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn shared_runtime() -> &'static Runtime {
+    static RUNTIME: OnceLock<Runtime> = OnceLock::new();
+    RUNTIME.get_or_init(|| {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .expect("build tokio runtime")
+    })
+}
+
+fuzz_target!(|data: &[u8]| {
+    if data.is_empty() {
+        return;
+    }
+    let mut u = Unstructured::new(data);
+    let ops: Vec<FuzzOp> = match Vec::<FuzzOp>::arbitrary(&mut u) {
+        Ok(ops) => ops.into_iter().take(MAX_OPS).collect(),
+        Err(_) => return,
+    };
+    if ops.is_empty() {
+        return;
+    }
+    let stream = OpStream::from_vec(ops.into_iter().map(to_op).collect());
+
+    let cfg = tiny_config();
+    let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+    let _ = shared_runtime().block_on(gauntlet.run_with_ops(stream));
+});

crates/tranquil-store/fuzz/fuzz_targets/metastore_key_codec.rs

@@ -0,0 +1,77 @@
+#![no_main]
+
+use arbitrary::Arbitrary;
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::metastore::encoding::{KeyBuilder, KeyReader};
+
+#[derive(Arbitrary, Debug, PartialEq, Eq)]
+enum Field {
+    U64(u64),
+    I64(i64),
+    U32(u32),
+    U16(u16),
+    Bool(bool),
+    Bytes(Vec<u8>),
+    String(String),
+}
+
+fn append(builder: KeyBuilder, field: &Field) -> KeyBuilder {
+    match field {
+        Field::U64(v) => builder.u64(*v),
+        Field::I64(v) => builder.i64(*v),
+        Field::U32(v) => builder.u32(*v),
+        Field::U16(v) => builder.u16(*v),
+        Field::Bool(v) => builder.bool(*v),
+        Field::Bytes(v) => builder.bytes(v),
+        Field::String(v) => builder.string(v),
+    }
+}
+
+fn consume(reader: &mut KeyReader<'_>, field: &Field) -> bool {
+    match field {
+        Field::U64(v) => reader.u64() == Some(*v),
+        Field::I64(v) => reader.i64() == Some(*v),
+        Field::U32(v) => reader.u32() == Some(*v),
+        Field::U16(v) => reader.u16() == Some(*v),
+        Field::Bool(v) => reader.bool() == Some(*v),
+        Field::Bytes(v) => reader.bytes().as_deref() == Some(v.as_slice()),
+        Field::String(v) => reader.string().as_deref() == Some(v.as_str()),
+    }
+}
+
+#[derive(Arbitrary, Debug)]
+enum Mode {
+    Roundtrip(Vec<Field>),
+    Raw(Vec<u8>),
+}
+
+fuzz_target!(|mode: Mode| {
+    match mode {
+        Mode::Roundtrip(fields) => {
+            let encoded1 = fields.iter().fold(KeyBuilder::new(), append).build();
+
+            let mut reader = KeyReader::new(encoded1.as_slice());
+            let all_match = fields.iter().all(|f| consume(&mut reader, f));
+            assert!(all_match, "roundtrip decode failed");
+            assert!(reader.is_empty(), "trailing bytes after decode");
+
+            let encoded2 = fields.iter().fold(KeyBuilder::new(), append).build();
+            assert_eq!(
+                encoded1.as_slice(),
+                encoded2.as_slice(),
+                "encoding not deterministic",
+            );
+        }
+        Mode::Raw(data) => {
+            let mut reader = KeyReader::new(&data);
+            let _ = reader.u64();
+            let _ = reader.i64();
+            let _ = reader.u32();
+            let _ = reader.u16();
+            let _ = reader.bool();
+            let _ = reader.bytes();
+            let _ = reader.string();
+            let _ = reader.tag();
+        }
+    }
+});

crates/tranquil-store/fuzz/fuzz_targets/segment_scan.rs

@@ -0,0 +1,31 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::eventlog::SegmentReader;
+
+const FUZZ_MAX_PAYLOAD: u32 = 1 << 20;
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/segment.tqe"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    if let Ok(reader) = SegmentReader::open(&sim, fd, FUZZ_MAX_PAYLOAD) {
+        reader.for_each(|_result| {});
+    }
+    let _ = sim.close(fd);
+});