From c30d73cd4da51ed4174f2e64ea0c24232a0040b7 Mon Sep 17 00:00:00 2001 From: Lewis Date: Sun, 19 Apr 2026 23:50:27 +0300 Subject: [PATCH] test(tranquil-store): migrate some tests to gauntlet Lewis: May this revision serve well! --- Cargo.lock | 61 ++- crates/tranquil-lexicon/src/dynamic.rs | 31 +- .../tests/gc_compaction_restart.rs | 2 +- crates/tranquil-store/src/lib.rs | 4 +- crates/tranquil-store/tests/eventlog_crash.rs | 12 +- crates/tranquil-store/tests/gauntlet_smoke.rs | 412 ++++++++++++++++-- crates/tranquil-store/tests/proptests.rs | 6 +- crates/tranquil-store/tests/sim_eventlog.rs | 23 +- 8 files changed, 461 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e00762..7ac7952 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "06b4f5ec222421e22bb0a8cbaa36b1d2b50fd45cdd30c915ded34108da78b29f" dependencies = [ "confique-macro", "serde", - "toml", + "toml 0.9.12+spec-1.1.0", ] [[package]] @@ -5050,7 +5050,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.5+spec-1.1.0", ] [[package]] @@ -6201,6 +6201,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.0.4" @@ -7144,6 +7153,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -7152,13 +7173,22 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ "indexmap 2.13.0", "serde_core", - "serde_spanned", + "serde_spanned 1.0.4", "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow 0.7.15", ] +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -7177,6 +7207,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.5+spec-1.1.0" @@ -7198,6 +7242,12 @@ dependencies = [ "winnow 1.0.0", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.0.7+spec-1.1.0" @@ -7866,6 +7916,7 @@ dependencies = [ "bytes", "chrono", "cid", + "clap", "dashmap", "fjall", "flume 0.11.1", @@ -7893,6 +7944,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "tokio", + "toml 0.8.23", "tracing", "tracing-subscriber", "tranquil-db", @@ -8854,6 +8906,9 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] [[package]] name = "winnow" diff --git a/crates/tranquil-lexicon/src/dynamic.rs b/crates/tranquil-lexicon/src/dynamic.rs index a9743c1..036226c 100644 --- a/crates/tranquil-lexicon/src/dynamic.rs +++ b/crates/tranquil-lexicon/src/dynamic.rs @@ -247,12 +247,10 @@ impl DynamicRegistry { self.wait_for_leader(nsid).await; match self.get_cached(nsid) { Some(doc) => Ok(doc), - None if self.is_negative_cached(nsid) => { - Err(ResolveError::NegativelyCached { - nsid: nsid.to_string(), - ttl_secs: NEGATIVE_CACHE_TTL.as_secs(), - }) - } + None if self.is_negative_cached(nsid) => Err(ResolveError::NegativelyCached { + nsid: nsid.to_string(), + ttl_secs: NEGATIVE_CACHE_TTL.as_secs(), + }), None => Err(ResolveError::LeaderAborted { nsid: nsid.to_string(), }), @@ -429,10 +427,7 @@ mod tests { let served = result.expect("stale entry must be served when refresh fails"); assert_eq!(served.id, "pet.nel.flaky"); assert!( - registry - .get_entry("pet.nel.flaky") - .unwrap() - .is_fresh(), + registry.get_entry("pet.nel.flaky").unwrap().is_fresh(), "failed refresh must bump expiry so subsequent lookups skip the resolver" ); assert!( @@ -492,12 +487,7 @@ mod tests { registry.insert_schema(doc); registry.expire_now("pet.nel.refresh"); - assert!( - !registry - .get_entry("pet.nel.refresh") - .unwrap() - .is_fresh() - ); + assert!(!registry.get_entry("pet.nel.refresh").unwrap().is_fresh()); let refreshed = registry .resolve_and_cache_with("pet.nel.refresh", |n| async move { @@ -512,10 +502,7 @@ mod tests { assert_eq!(refreshed.id, "pet.nel.refresh"); assert!( - registry - .get_entry("pet.nel.refresh") - .unwrap() - .is_fresh(), + registry.get_entry("pet.nel.refresh").unwrap().is_fresh(), "refresh must restore freshness" ); } @@ -601,9 +588,7 @@ mod tests { assert!(registry.is_negative_cached("pet.nel.failHerd")); } - async fn futures_collect( - handles: Vec>, - ) -> Vec { + async fn futures_collect(handles: Vec>) -> Vec { futures::future::join_all(handles) .await .into_iter() diff --git a/crates/tranquil-pds/tests/gc_compaction_restart.rs b/crates/tranquil-pds/tests/gc_compaction_restart.rs index f351c73..c993820 100644 --- a/crates/tranquil-pds/tests/gc_compaction_restart.rs +++ b/crates/tranquil-pds/tests/gc_compaction_restart.rs @@ -111,7 +111,7 @@ async fn mst_blocks_survive_full_store_reopen() { let max_file_size = store .list_data_files() .ok() - .and_then(|_| Some(4 * 1024 * 1024u64)) + .map(|_| 4 * 1024 * 1024u64) .unwrap_or(4 * 1024 * 1024); let reopened_missing = tokio::task::spawn_blocking(move || { diff --git a/crates/tranquil-store/src/lib.rs b/crates/tranquil-store/src/lib.rs index 0abad04..178702c 100644 --- a/crates/tranquil-store/src/lib.rs +++ b/crates/tranquil-store/src/lib.rs @@ -28,8 +28,8 @@ pub use record::{ }; #[cfg(any(test, feature = "test-harness"))] pub use sim::{ - FaultConfig, OpRecord, SimulatedIO, sim_proptest_cases, sim_seed_count, sim_seed_range, - sim_single_seed, + FaultConfig, LatencyNs, OpRecord, Probability, SimulatedIO, SyncReorderWindow, + sim_proptest_cases, sim_seed_count, sim_seed_range, sim_single_seed, }; pub(crate) fn wall_clock_ms() -> blockstore::WallClockMs { diff --git a/crates/tranquil-store/tests/eventlog_crash.rs b/crates/tranquil-store/tests/eventlog_crash.rs index d706393..f910e61 100644 --- a/crates/tranquil-store/tests/eventlog_crash.rs +++ b/crates/tranquil-store/tests/eventlog_crash.rs @@ -7,7 +7,9 @@ use tranquil_store::eventlog::{ SEGMENT_HEADER_SIZE, SegmentId, SegmentManager, SegmentReader, SegmentWriter, TimestampMicros, ValidEvent, rebuild_from_segment, }; -use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO, sim_seed_range}; +use tranquil_store::{ + FaultConfig, OpenOptions, Probability, SimulatedIO, StorageIO, sim_seed_range, +}; fn setup_manager(sim: SimulatedIO, max_segment_size: u64) -> Arc> { Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap()) @@ -540,15 +542,15 @@ fn fault_configs() -> Vec<(&'static str, FaultConfig)> { ( "partial_writes_only", FaultConfig { - partial_write_probability: 0.15, + partial_write_probability: Probability::new(0.15), ..FaultConfig::none() }, ), ( "sync_failures_only", FaultConfig { - sync_failure_probability: 0.10, - dir_sync_failure_probability: 0.05, + sync_failure_probability: Probability::new(0.10), + dir_sync_failure_probability: Probability::new(0.05), ..FaultConfig::none() }, ), @@ -556,7 +558,7 @@ fn fault_configs() -> Vec<(&'static str, FaultConfig)> { ( "bit_flips_only", FaultConfig { - bit_flip_on_read_probability: 0.05, + bit_flip_on_read_probability: Probability::new(0.05), ..FaultConfig::none() }, ), diff --git a/crates/tranquil-store/tests/gauntlet_smoke.rs b/crates/tranquil-store/tests/gauntlet_smoke.rs index 1037979..1810368 100644 --- a/crates/tranquil-store/tests/gauntlet_smoke.rs +++ b/crates/tranquil-store/tests/gauntlet_smoke.rs @@ -1,10 +1,23 @@ +use tranquil_store::FaultConfig; use tranquil_store::blockstore::GroupCommitConfig; use tranquil_store::gauntlet::{ - CollectionName, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize, MaxFileSize, - OpCount, OpInterval, OpWeights, RestartPolicy, RunLimits, Scenario, Seed, ShardCount, - SizeDistribution, StoreConfig, ValueBytes, WallMs, WorkloadModel, config_for, farm, + CollectionName, ConfigOverrides, DidSpaceSize, Gauntlet, GauntletConfig, GauntletReport, + InvariantSet, IoBackend, KeySpaceSize, MaxFileSize, OpCount, OpInterval, OpWeights, + RegressionRecord, RestartPolicy, RetentionMaxSecs, RunLimits, Scenario, Seed, ShardCount, + SizeDistribution, StoreConfig, StoreOverrides, ValueBytes, WallMs, WorkloadModel, + WriterConcurrency, config_for, farm, }; +#[track_caller] +fn assert_clean(report: &GauntletReport) { + let violations: Vec = report + .violations + .iter() + .map(|v| format!("{}: {}", v.invariant, v.detail)) + .collect(); + assert!(report.is_clean(), "violations: {violations:?}"); +} + #[test] #[ignore = "long running, 30 seeds of 10k ops each"] fn smoke_pr_30_seeds() { @@ -38,13 +51,15 @@ fn fast_sanity_config(seed: Seed) -> GauntletConfig { workload: WorkloadModel { weights: OpWeights { add: 80, - delete: 0, compact: 10, checkpoint: 10, + ..OpWeights::default() }, size_distribution: SizeDistribution::Fixed(ValueBytes(64)), collections: vec![CollectionName("app.bsky.feed.post".to_string())], key_space: KeySpaceSize(100), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(3600), }, op_count: OpCount(200), invariants: InvariantSet::REFCOUNT_CONSERVATION @@ -65,6 +80,8 @@ fn fast_sanity_config(seed: Seed) -> GauntletConfig { }, shard_count: ShardCount(1), }, + eventlog: None, + writer_concurrency: WriterConcurrency(1), } } @@ -74,20 +91,8 @@ async fn gauntlet_fast_sanity() { .expect("build gauntlet") .run() .await; - assert!( - report.is_clean(), - "violations: {:?}", - report - .violations - .iter() - .map(|v| format!("{}: {}", v.invariant, v.detail)) - .collect::>() - ); - assert!( - report.restarts.0 >= 2, - "expected at least 2 restarts, got {}", - report.restarts.0 - ); + assert_clean(&report); + assert!(report.restarts.0 >= 2); assert_eq!(report.ops_executed.0, 200); } @@ -95,38 +100,363 @@ async fn gauntlet_fast_sanity() { async fn full_stack_restart_port() { let cfg = config_for(Scenario::FullStackRestart, Seed(1)); let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; - assert!( - report.is_clean(), - "violations: {:?}", - report - .violations - .iter() - .map(|v| format!("{}: {}", v.invariant, v.detail)) - .collect::>() - ); + assert_clean(&report); assert_eq!( report.restarts.0, 10, "FullStackRestart with EveryNOps(500) over 5000 ops must restart exactly 10 times", ); } +#[tokio::test] +async fn compaction_idempotent_sanity() { + let cfg = GauntletConfig { + seed: Seed(3), + io: IoBackend::Real, + workload: WorkloadModel { + weights: OpWeights { + add: 70, + delete: 10, + compact: 15, + checkpoint: 5, + ..OpWeights::default() + }, + size_distribution: SizeDistribution::Fixed(ValueBytes(64)), + collections: vec![CollectionName("app.bsky.feed.post".to_string())], + key_space: KeySpaceSize(50), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(3600), + }, + op_count: OpCount(300), + invariants: InvariantSet::REFCOUNT_CONSERVATION + | InvariantSet::REACHABILITY + | InvariantSet::READ_AFTER_WRITE + | InvariantSet::COMPACTION_IDEMPOTENT, + limits: RunLimits { + max_wall_ms: Some(WallMs(30_000)), + }, + restart_policy: RestartPolicy::Never, + store: StoreConfig { + max_file_size: MaxFileSize(4096), + group_commit: GroupCommitConfig::default(), + shard_count: ShardCount(1), + }, + eventlog: None, + writer_concurrency: WriterConcurrency(1), + }; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); +} + +#[tokio::test] +async fn no_orphan_files_sanity() { + let cfg = GauntletConfig { + seed: Seed(11), + io: IoBackend::Real, + workload: WorkloadModel { + weights: OpWeights { + add: 90, + checkpoint: 10, + ..OpWeights::default() + }, + size_distribution: SizeDistribution::Fixed(ValueBytes(128)), + collections: vec![CollectionName("app.bsky.feed.post".to_string())], + key_space: KeySpaceSize(80), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(3600), + }, + op_count: OpCount(200), + invariants: InvariantSet::REFCOUNT_CONSERVATION + | InvariantSet::REACHABILITY + | InvariantSet::READ_AFTER_WRITE + | InvariantSet::COMPACTION_IDEMPOTENT + | InvariantSet::NO_ORPHAN_FILES, + limits: RunLimits { + max_wall_ms: Some(WallMs(30_000)), + }, + restart_policy: RestartPolicy::Never, + store: StoreConfig { + max_file_size: MaxFileSize(64 * 1024), + group_commit: GroupCommitConfig::default(), + shard_count: ShardCount(1), + }, + eventlog: None, + writer_concurrency: WriterConcurrency(1), + }; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); +} + +#[tokio::test] +async fn simulated_pristine_roundtrip() { + let cfg = GauntletConfig { + seed: Seed(21), + io: IoBackend::Simulated { + fault: FaultConfig::none(), + }, + workload: WorkloadModel { + weights: OpWeights { + add: 80, + delete: 10, + compact: 5, + checkpoint: 5, + ..OpWeights::default() + }, + size_distribution: SizeDistribution::Fixed(ValueBytes(96)), + collections: vec![CollectionName("app.bsky.feed.post".to_string())], + key_space: KeySpaceSize(80), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(3600), + }, + op_count: OpCount(300), + invariants: InvariantSet::REFCOUNT_CONSERVATION + | InvariantSet::REACHABILITY + | InvariantSet::ACKED_WRITE_PERSISTENCE + | InvariantSet::READ_AFTER_WRITE + | InvariantSet::RESTART_IDEMPOTENT + | InvariantSet::CHECKSUM_COVERAGE, + limits: RunLimits { + max_wall_ms: Some(WallMs(60_000)), + }, + restart_policy: RestartPolicy::EveryNOps(OpInterval(100)), + store: StoreConfig { + max_file_size: MaxFileSize(8 * 1024), + group_commit: GroupCommitConfig::default(), + shard_count: ShardCount(1), + }, + eventlog: None, + writer_concurrency: WriterConcurrency(1), + }; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); + assert_eq!(report.ops_executed.0, 300); + assert!(report.restarts.0 >= 2); +} + +#[tokio::test] +async fn firehose_fanout_pristine_smoke() { + use tranquil_store::gauntlet::{EventLogConfig, MaxSegmentSize}; + + let cfg = GauntletConfig { + seed: Seed(1), + io: IoBackend::Simulated { + fault: FaultConfig::none(), + }, + workload: WorkloadModel { + weights: OpWeights { + add: 20, + compact: 2, + checkpoint: 3, + append_event: 60, + sync_event_log: 10, + run_retention: 5, + ..OpWeights::default() + }, + size_distribution: SizeDistribution::Fixed(ValueBytes(128)), + collections: vec![CollectionName("app.bsky.feed.post".to_string())], + key_space: KeySpaceSize(100), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(60), + }, + op_count: OpCount(2_000), + invariants: InvariantSet::REFCOUNT_CONSERVATION + | InvariantSet::REACHABILITY + | InvariantSet::ACKED_WRITE_PERSISTENCE + | InvariantSet::READ_AFTER_WRITE + | InvariantSet::RESTART_IDEMPOTENT + | InvariantSet::MONOTONIC_SEQ + | InvariantSet::FSYNC_ORDERING + | InvariantSet::TOMBSTONE_BOUND, + limits: RunLimits { + max_wall_ms: Some(WallMs(60_000)), + }, + restart_policy: RestartPolicy::EveryNOps(OpInterval(500)), + store: StoreConfig { + max_file_size: MaxFileSize(16 * 1024), + group_commit: GroupCommitConfig::default(), + shard_count: ShardCount(1), + }, + eventlog: Some(EventLogConfig { + max_segment_size: MaxSegmentSize(32 * 1024), + }), + writer_concurrency: WriterConcurrency(1), + }; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); + assert_eq!(report.ops_executed.0, 2_000); + assert!(report.restarts.0 >= 2); +} + +#[tokio::test] +async fn contended_readers_pristine_smoke() { + let cfg = GauntletConfig { + seed: Seed(1), + io: IoBackend::Simulated { + fault: FaultConfig::none(), + }, + workload: WorkloadModel { + weights: OpWeights { + add: 20, + compact: 2, + checkpoint: 3, + read_record: 60, + read_block: 15, + ..OpWeights::default() + }, + size_distribution: SizeDistribution::Fixed(ValueBytes(128)), + collections: vec![CollectionName("app.bsky.feed.post".to_string())], + key_space: KeySpaceSize(200), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(3600), + }, + op_count: OpCount(1_000), + invariants: InvariantSet::REFCOUNT_CONSERVATION + | InvariantSet::REACHABILITY + | InvariantSet::ACKED_WRITE_PERSISTENCE + | InvariantSet::READ_AFTER_WRITE + | InvariantSet::RESTART_IDEMPOTENT, + limits: RunLimits { + max_wall_ms: Some(WallMs(60_000)), + }, + restart_policy: RestartPolicy::EveryNOps(OpInterval(250)), + store: StoreConfig { + max_file_size: MaxFileSize(16 * 1024), + group_commit: GroupCommitConfig::default(), + shard_count: ShardCount(1), + }, + eventlog: None, + writer_concurrency: WriterConcurrency(16), + }; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); + assert_eq!(report.ops_executed.0, 1_000); + assert!(report.restarts.0 >= 2); +} + +#[tokio::test] +async fn contended_writers_pristine_smoke() { + let cfg = GauntletConfig { + seed: Seed(2), + io: IoBackend::Simulated { + fault: FaultConfig::none(), + }, + workload: WorkloadModel { + weights: OpWeights { + add: 85, + delete: 5, + compact: 3, + checkpoint: 2, + read_record: 4, + read_block: 1, + ..OpWeights::default() + }, + size_distribution: SizeDistribution::Fixed(ValueBytes(128)), + collections: vec![CollectionName("app.bsky.feed.post".to_string())], + key_space: KeySpaceSize(500), + did_space: DidSpaceSize(32), + retention_max_secs: RetentionMaxSecs(3600), + }, + op_count: OpCount(1_000), + invariants: InvariantSet::REFCOUNT_CONSERVATION + | InvariantSet::REACHABILITY + | InvariantSet::ACKED_WRITE_PERSISTENCE + | InvariantSet::READ_AFTER_WRITE + | InvariantSet::RESTART_IDEMPOTENT, + limits: RunLimits { + max_wall_ms: Some(WallMs(60_000)), + }, + restart_policy: RestartPolicy::EveryNOps(OpInterval(250)), + store: StoreConfig { + max_file_size: MaxFileSize(16 * 1024), + group_commit: GroupCommitConfig::default(), + shard_count: ShardCount(1), + }, + eventlog: None, + writer_concurrency: WriterConcurrency(8), + }; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); + assert_eq!(report.ops_executed.0, 1_000); + assert!(report.restarts.0 >= 2); +} + +#[tokio::test] +async fn report_carries_generated_ops_when_clean() { + let cfg = fast_sanity_config(Seed(5)); + let expected_len = cfg.op_count.0; + let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; + assert_clean(&report); + assert_eq!( + report.ops.len(), + expected_len, + "clean report missing op stream" + ); +} + +#[tokio::test] +async fn regression_round_trip_replays_injected_ops() { + let overrides = ConfigOverrides { + op_count: Some(25), + store: StoreOverrides { + max_file_size: Some(8192), + ..StoreOverrides::default() + }, + ..ConfigOverrides::default() + }; + let mut cfg = config_for(Scenario::SmokePR, Seed(99)); + overrides.apply_to(&mut cfg); + + let original_report = Gauntlet::new(cfg.clone()) + .expect("build gauntlet") + .run() + .await; + let captured_ops = original_report.ops.clone(); + assert_eq!( + captured_ops.len(), + 25, + "captured op stream must match op_count override" + ); + + let dir = tempfile::TempDir::new().unwrap(); + let record = RegressionRecord::from_report( + Scenario::SmokePR, + overrides.clone(), + &original_report, + captured_ops.len(), + captured_ops.clone(), + ); + let written = record.write_to(dir.path()).expect("write regression"); + let loaded = RegressionRecord::load(&written).expect("load regression"); + assert_eq!(loaded.overrides, overrides); + assert_eq!(loaded.ops.len(), captured_ops.len()); + + let rebuilt = loaded.build_config().expect("rebuild config"); + assert_eq!(rebuilt.op_count.0, 25); + assert_eq!(rebuilt.store.max_file_size.0, 8192); + + let replay = Gauntlet::new(rebuilt) + .expect("build gauntlet") + .run_with_ops(loaded.op_stream()) + .await; + assert_eq!( + replay.violations.len(), + original_report.violations.len(), + "replay from regression must produce same violation count", + ); + let original_inv: Vec<&'static str> = original_report + .violations + .iter() + .map(|v| v.invariant) + .collect(); + let replay_inv: Vec<&'static str> = replay.violations.iter().map(|v| v.invariant).collect(); + assert_eq!(original_inv, replay_inv); + assert_eq!(replay.ops.len(), captured_ops.len()); +} + #[tokio::test] #[ignore = "long running, 100k ops with around 20 restarts"] async fn mst_restart_churn_single_seed() { let cfg = config_for(Scenario::MstRestartChurn, Seed(42)); let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; - assert!( - report.is_clean(), - "violations: {:?}", - report - .violations - .iter() - .map(|v| format!("{}: {}", v.invariant, v.detail)) - .collect::>() - ); - assert!( - report.restarts.0 >= 1, - "PoissonByOps(5000) over 100k ops should fire at least 1 restart, got {}", - report.restarts.0 - ); + assert_clean(&report); + assert!(report.restarts.0 >= 1); } diff --git a/crates/tranquil-store/tests/proptests.rs b/crates/tranquil-store/tests/proptests.rs index 9f50161..5958256 100644 --- a/crates/tranquil-store/tests/proptests.rs +++ b/crates/tranquil-store/tests/proptests.rs @@ -2,8 +2,8 @@ use proptest::prelude::*; use std::path::Path; use tranquil_store::{ - FaultConfig, HEADER_SIZE, OpenOptions, ReadRecord, RecordReader, RecordWriter, SimulatedIO, - StorageIO, run_crash_test, run_pristine_comparison, sim_proptest_cases, + FaultConfig, HEADER_SIZE, OpenOptions, Probability, ReadRecord, RecordReader, RecordWriter, + SimulatedIO, StorageIO, run_crash_test, run_pristine_comparison, sim_proptest_cases, }; fn arb_payloads(max_count: usize, max_size: usize) -> BoxedStrategy>> { @@ -151,7 +151,7 @@ proptest! { data in proptest::collection::vec(any::(), 64..4096), ) { let config = FaultConfig { - partial_write_probability: 0.5, + partial_write_probability: Probability::new(0.5), ..FaultConfig::none() }; let dir = Path::new("/test"); diff --git a/crates/tranquil-store/tests/sim_eventlog.rs b/crates/tranquil-store/tests/sim_eventlog.rs index 8a2f9da..0637a8f 100644 --- a/crates/tranquil-store/tests/sim_eventlog.rs +++ b/crates/tranquil-store/tests/sim_eventlog.rs @@ -8,7 +8,7 @@ use tranquil_store::eventlog::{ DidHash, EVENT_RECORD_OVERHEAD, EventLogWriter, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SEGMENT_HEADER_SIZE, SegmentId, SegmentManager, SegmentReader, ValidEvent, }; -use tranquil_store::{FaultConfig, SimulatedIO, StorageIO, sim_seed_range}; +use tranquil_store::{FaultConfig, Probability, SimulatedIO, StorageIO, sim_seed_range}; use common::Rng; @@ -529,8 +529,8 @@ fn group_sync_crash_after_append_before_sync() { fn group_sync_crash_mid_sync_partial_fsync() { sim_seed_range().into_par_iter().for_each(|seed| { let fault_config = FaultConfig { - sync_failure_probability: 0.3, - partial_write_probability: 0.1, + sync_failure_probability: Probability::new(0.3), + partial_write_probability: Probability::new(0.1), ..FaultConfig::none() }; let sim = SimulatedIO::new(seed, fault_config); @@ -682,9 +682,9 @@ fn group_sync_no_double_sync_no_skipped_events() { fn group_sync_contention_under_faults() { sim_seed_range().into_par_iter().for_each(|seed| { let fault_config = FaultConfig { - partial_write_probability: 0.05, - sync_failure_probability: 0.10, - dir_sync_failure_probability: 0.05, + partial_write_probability: Probability::new(0.05), + sync_failure_probability: Probability::new(0.10), + dir_sync_failure_probability: Probability::new(0.05), ..FaultConfig::none() }; let sim = SimulatedIO::new(seed, fault_config); @@ -897,12 +897,11 @@ fn multi_rotation_crash_at_each_phase() { #[test] fn aggressive_faults_group_sync_recovery() { let fault_config = FaultConfig { - partial_write_probability: 0.15, - sync_failure_probability: 0.10, - dir_sync_failure_probability: 0.05, - misdirected_write_probability: 0.05, - bit_flip_on_read_probability: 0.0, - io_error_probability: 0.0, + partial_write_probability: Probability::new(0.15), + sync_failure_probability: Probability::new(0.10), + dir_sync_failure_probability: Probability::new(0.05), + misdirected_write_probability: Probability::new(0.05), + ..FaultConfig::none() }; sim_seed_range().into_par_iter().for_each(|seed| {