mirrors/tranquil-pds
1
0
Fork 0
mirror of https://tangled.org/tranquil.farm/tranquil-pds synced 2026-04-24 10:20:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(tranquil-store): migrate some tests to gauntlet

Browse Source 
Lewis: May this revision serve well! <lu5a@proton.me>
This commit is contained in:
Lewis
2026-04-19 23:50:27 +03:00
parent 0fab8f2eb9
commit c30d73cd4d
8 changed files with 461 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
Cargo.lock generated
View File

@@ -1558,7 +1558,7 @@ checksum = "06b4f5ec222421e22bb0a8cbaa36b1d2b50fd45cdd30c915ded34108da78b29f"
dependencies = [
"confique-macro",
"serde",
"toml",
"toml 0.9.12+spec-1.1.0",
]
[[package]]
@@ -5050,7 +5050,7 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
dependencies = [
"toml_edit",
"toml_edit 0.25.5+spec-1.1.0",
]
[[package]]
@@ -6201,6 +6201,15 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "serde_spanned"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "1.0.4"
@@ -7144,6 +7153,18 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
"toml_edit 0.22.27",
]
[[package]]
name = "toml"
version = "0.9.12+spec-1.1.0"
@@ -7152,13 +7173,22 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
dependencies = [
"indexmap 2.13.0",
"serde_core",
"serde_spanned",
"serde_spanned 1.0.4",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"toml_writer",
"winnow 0.7.15",
]
[[package]]
name = "toml_datetime"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
[[package]]
name = "toml_datetime"
version = "0.7.5+spec-1.1.0"
@@ -7177,6 +7207,20 @@ dependencies = [
"serde_core",
]
[[package]]
name = "toml_edit"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.13.0",
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
"toml_write",
"winnow 0.7.15",
]
[[package]]
name = "toml_edit"
version = "0.25.5+spec-1.1.0"
@@ -7198,6 +7242,12 @@ dependencies = [
"winnow 1.0.0",
]
[[package]]
name = "toml_write"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "toml_writer"
version = "1.0.7+spec-1.1.0"
@@ -7866,6 +7916,7 @@ dependencies = [
"bytes",
"chrono",
"cid",
"clap",
"dashmap",
"fjall",
"flume 0.11.1",
@@ -7893,6 +7944,7 @@ dependencies = [
"thiserror 2.0.18",
"tikv-jemallocator",
"tokio",
"toml 0.8.23",
"tracing",
"tracing-subscriber",
"tranquil-db",
@@ -8854,6 +8906,9 @@ name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
dependencies = [
"memchr",
]
[[package]]
name = "winnow"

31
crates/tranquil-lexicon/src/dynamic.rs
View File

@@ -247,12 +247,10 @@ impl DynamicRegistry {
self.wait_for_leader(nsid).await;
match self.get_cached(nsid) {
Some(doc) => Ok(doc),
None if self.is_negative_cached(nsid) => {
Err(ResolveError::NegativelyCached {
nsid: nsid.to_string(),
ttl_secs: NEGATIVE_CACHE_TTL.as_secs(),
})
}
None if self.is_negative_cached(nsid) => Err(ResolveError::NegativelyCached {
nsid: nsid.to_string(),
ttl_secs: NEGATIVE_CACHE_TTL.as_secs(),
}),
None => Err(ResolveError::LeaderAborted {
nsid: nsid.to_string(),
}),
@@ -429,10 +427,7 @@ mod tests {
let served = result.expect("stale entry must be served when refresh fails");
assert_eq!(served.id, "pet.nel.flaky");
assert!(
registry
.get_entry("pet.nel.flaky")
.unwrap()
.is_fresh(),
registry.get_entry("pet.nel.flaky").unwrap().is_fresh(),
"failed refresh must bump expiry so subsequent lookups skip the resolver"
);
assert!(
@@ -492,12 +487,7 @@ mod tests {
registry.insert_schema(doc);
registry.expire_now("pet.nel.refresh");
assert!(
!registry
.get_entry("pet.nel.refresh")
.unwrap()
.is_fresh()
);
assert!(!registry.get_entry("pet.nel.refresh").unwrap().is_fresh());
let refreshed = registry
.resolve_and_cache_with("pet.nel.refresh", |n| async move {
@@ -512,10 +502,7 @@ mod tests {
assert_eq!(refreshed.id, "pet.nel.refresh");
assert!(
registry
.get_entry("pet.nel.refresh")
.unwrap()
.is_fresh(),
registry.get_entry("pet.nel.refresh").unwrap().is_fresh(),
"refresh must restore freshness"
);
}
@@ -601,9 +588,7 @@ mod tests {
assert!(registry.is_negative_cached("pet.nel.failHerd"));
}
async fn futures_collect<T>(
handles: Vec<tokio::task::JoinHandle<T>>,
) -> Vec<T> {
async fn futures_collect<T>(handles: Vec<tokio::task::JoinHandle<T>>) -> Vec<T> {
futures::future::join_all(handles)
.await
.into_iter()

2
crates/tranquil-pds/tests/gc_compaction_restart.rs
View File

@@ -111,7 +111,7 @@ async fn mst_blocks_survive_full_store_reopen() {
let max_file_size = store
.list_data_files()
.ok()
.and_then(|_| Some(4 * 1024 * 1024u64))
.map(|_| 4 * 1024 * 1024u64)
.unwrap_or(4 * 1024 * 1024);
let reopened_missing = tokio::task::spawn_blocking(move || {

4
crates/tranquil-store/src/lib.rs
View File

@@ -28,8 +28,8 @@ pub use record::{
};
#[cfg(any(test, feature = "test-harness"))]
pub use sim::{
FaultConfig, OpRecord, SimulatedIO, sim_proptest_cases, sim_seed_count, sim_seed_range,
sim_single_seed,
FaultConfig, LatencyNs, OpRecord, Probability, SimulatedIO, SyncReorderWindow,
sim_proptest_cases, sim_seed_count, sim_seed_range, sim_single_seed,
};
pub(crate) fn wall_clock_ms() -> blockstore::WallClockMs {

12
crates/tranquil-store/tests/eventlog_crash.rs
View File

@@ -7,7 +7,9 @@ use tranquil_store::eventlog::{
SEGMENT_HEADER_SIZE, SegmentId, SegmentManager, SegmentReader, SegmentWriter, TimestampMicros,
ValidEvent, rebuild_from_segment,
};
use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO, sim_seed_range};
use tranquil_store::{
FaultConfig, OpenOptions, Probability, SimulatedIO, StorageIO, sim_seed_range,
};
fn setup_manager(sim: SimulatedIO, max_segment_size: u64) -> Arc<SegmentManager<SimulatedIO>> {
Arc::new(SegmentManager::new(sim, PathBuf::from("/segments"), max_segment_size).unwrap())
@@ -540,15 +542,15 @@ fn fault_configs() -> Vec<(&'static str, FaultConfig)> {
(
"partial_writes_only",
FaultConfig {
partial_write_probability: 0.15,
partial_write_probability: Probability::new(0.15),
..FaultConfig::none()
},
),
(
"sync_failures_only",
FaultConfig {
sync_failure_probability: 0.10,
dir_sync_failure_probability: 0.05,
sync_failure_probability: Probability::new(0.10),
dir_sync_failure_probability: Probability::new(0.05),
..FaultConfig::none()
},
),
@@ -556,7 +558,7 @@ fn fault_configs() -> Vec<(&'static str, FaultConfig)> {
(
"bit_flips_only",
FaultConfig {
bit_flip_on_read_probability: 0.05,
bit_flip_on_read_probability: Probability::new(0.05),
..FaultConfig::none()
},
),

412
crates/tranquil-store/tests/gauntlet_smoke.rs
View File

@@ -1,10 +1,23 @@
use tranquil_store::FaultConfig;
use tranquil_store::blockstore::GroupCommitConfig;
use tranquil_store::gauntlet::{
CollectionName, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize, MaxFileSize,
OpCount, OpInterval, OpWeights, RestartPolicy, RunLimits, Scenario, Seed, ShardCount,
SizeDistribution, StoreConfig, ValueBytes, WallMs, WorkloadModel, config_for, farm,
CollectionName, ConfigOverrides, DidSpaceSize, Gauntlet, GauntletConfig, GauntletReport,
InvariantSet, IoBackend, KeySpaceSize, MaxFileSize, OpCount, OpInterval, OpWeights,
RegressionRecord, RestartPolicy, RetentionMaxSecs, RunLimits, Scenario, Seed, ShardCount,
SizeDistribution, StoreConfig, StoreOverrides, ValueBytes, WallMs, WorkloadModel,
WriterConcurrency, config_for, farm,
};
#[track_caller]
fn assert_clean(report: &GauntletReport) {
let violations: Vec<String> = report
.violations
.iter()
.map(|v| format!("{}: {}", v.invariant, v.detail))
.collect();
assert!(report.is_clean(), "violations: {violations:?}");
}
#[test]
#[ignore = "long running, 30 seeds of 10k ops each"]
fn smoke_pr_30_seeds() {
@@ -38,13 +51,15 @@ fn fast_sanity_config(seed: Seed) -> GauntletConfig {
workload: WorkloadModel {
weights: OpWeights {
add: 80,
delete: 0,
compact: 10,
checkpoint: 10,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(100),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(3600),
},
op_count: OpCount(200),
invariants: InvariantSet::REFCOUNT_CONSERVATION
@@ -65,6 +80,8 @@ fn fast_sanity_config(seed: Seed) -> GauntletConfig {
},
shard_count: ShardCount(1),
},
eventlog: None,
writer_concurrency: WriterConcurrency(1),
}
}
@@ -74,20 +91,8 @@ async fn gauntlet_fast_sanity() {
.expect("build gauntlet")
.run()
.await;
assert!(
report.is_clean(),
"violations: {:?}",
report
.violations
.iter()
.map(|v| format!("{}: {}", v.invariant, v.detail))
.collect::<Vec<_>>()
);
assert!(
report.restarts.0 >= 2,
"expected at least 2 restarts, got {}",
report.restarts.0
);
assert_clean(&report);
assert!(report.restarts.0 >= 2);
assert_eq!(report.ops_executed.0, 200);
}
@@ -95,38 +100,363 @@ async fn gauntlet_fast_sanity() {
async fn full_stack_restart_port() {
let cfg = config_for(Scenario::FullStackRestart, Seed(1));
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert!(
report.is_clean(),
"violations: {:?}",
report
.violations
.iter()
.map(|v| format!("{}: {}", v.invariant, v.detail))
.collect::<Vec<_>>()
);
assert_clean(&report);
assert_eq!(
report.restarts.0, 10,
"FullStackRestart with EveryNOps(500) over 5000 ops must restart exactly 10 times",
);
}
#[tokio::test]
async fn compaction_idempotent_sanity() {
let cfg = GauntletConfig {
seed: Seed(3),
io: IoBackend::Real,
workload: WorkloadModel {
weights: OpWeights {
add: 70,
delete: 10,
compact: 15,
checkpoint: 5,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(50),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(3600),
},
op_count: OpCount(300),
invariants: InvariantSet::REFCOUNT_CONSERVATION
| InvariantSet::REACHABILITY
| InvariantSet::READ_AFTER_WRITE
| InvariantSet::COMPACTION_IDEMPOTENT,
limits: RunLimits {
max_wall_ms: Some(WallMs(30_000)),
},
restart_policy: RestartPolicy::Never,
store: StoreConfig {
max_file_size: MaxFileSize(4096),
group_commit: GroupCommitConfig::default(),
shard_count: ShardCount(1),
},
eventlog: None,
writer_concurrency: WriterConcurrency(1),
};
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
}
#[tokio::test]
async fn no_orphan_files_sanity() {
let cfg = GauntletConfig {
seed: Seed(11),
io: IoBackend::Real,
workload: WorkloadModel {
weights: OpWeights {
add: 90,
checkpoint: 10,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(80),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(3600),
},
op_count: OpCount(200),
invariants: InvariantSet::REFCOUNT_CONSERVATION
| InvariantSet::REACHABILITY
| InvariantSet::READ_AFTER_WRITE
| InvariantSet::COMPACTION_IDEMPOTENT
| InvariantSet::NO_ORPHAN_FILES,
limits: RunLimits {
max_wall_ms: Some(WallMs(30_000)),
},
restart_policy: RestartPolicy::Never,
store: StoreConfig {
max_file_size: MaxFileSize(64 * 1024),
group_commit: GroupCommitConfig::default(),
shard_count: ShardCount(1),
},
eventlog: None,
writer_concurrency: WriterConcurrency(1),
};
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
}
#[tokio::test]
async fn simulated_pristine_roundtrip() {
let cfg = GauntletConfig {
seed: Seed(21),
io: IoBackend::Simulated {
fault: FaultConfig::none(),
},
workload: WorkloadModel {
weights: OpWeights {
add: 80,
delete: 10,
compact: 5,
checkpoint: 5,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(96)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(80),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(3600),
},
op_count: OpCount(300),
invariants: InvariantSet::REFCOUNT_CONSERVATION
| InvariantSet::REACHABILITY
| InvariantSet::ACKED_WRITE_PERSISTENCE
| InvariantSet::READ_AFTER_WRITE
| InvariantSet::RESTART_IDEMPOTENT
| InvariantSet::CHECKSUM_COVERAGE,
limits: RunLimits {
max_wall_ms: Some(WallMs(60_000)),
},
restart_policy: RestartPolicy::EveryNOps(OpInterval(100)),
store: StoreConfig {
max_file_size: MaxFileSize(8 * 1024),
group_commit: GroupCommitConfig::default(),
shard_count: ShardCount(1),
},
eventlog: None,
writer_concurrency: WriterConcurrency(1),
};
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
assert_eq!(report.ops_executed.0, 300);
assert!(report.restarts.0 >= 2);
}
#[tokio::test]
async fn firehose_fanout_pristine_smoke() {
use tranquil_store::gauntlet::{EventLogConfig, MaxSegmentSize};
let cfg = GauntletConfig {
seed: Seed(1),
io: IoBackend::Simulated {
fault: FaultConfig::none(),
},
workload: WorkloadModel {
weights: OpWeights {
add: 20,
compact: 2,
checkpoint: 3,
append_event: 60,
sync_event_log: 10,
run_retention: 5,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(100),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(60),
},
op_count: OpCount(2_000),
invariants: InvariantSet::REFCOUNT_CONSERVATION
| InvariantSet::REACHABILITY
| InvariantSet::ACKED_WRITE_PERSISTENCE
| InvariantSet::READ_AFTER_WRITE
| InvariantSet::RESTART_IDEMPOTENT
| InvariantSet::MONOTONIC_SEQ
| InvariantSet::FSYNC_ORDERING
| InvariantSet::TOMBSTONE_BOUND,
limits: RunLimits {
max_wall_ms: Some(WallMs(60_000)),
},
restart_policy: RestartPolicy::EveryNOps(OpInterval(500)),
store: StoreConfig {
max_file_size: MaxFileSize(16 * 1024),
group_commit: GroupCommitConfig::default(),
shard_count: ShardCount(1),
},
eventlog: Some(EventLogConfig {
max_segment_size: MaxSegmentSize(32 * 1024),
}),
writer_concurrency: WriterConcurrency(1),
};
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
assert_eq!(report.ops_executed.0, 2_000);
assert!(report.restarts.0 >= 2);
}
#[tokio::test]
async fn contended_readers_pristine_smoke() {
let cfg = GauntletConfig {
seed: Seed(1),
io: IoBackend::Simulated {
fault: FaultConfig::none(),
},
workload: WorkloadModel {
weights: OpWeights {
add: 20,
compact: 2,
checkpoint: 3,
read_record: 60,
read_block: 15,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(200),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(3600),
},
op_count: OpCount(1_000),
invariants: InvariantSet::REFCOUNT_CONSERVATION
| InvariantSet::REACHABILITY
| InvariantSet::ACKED_WRITE_PERSISTENCE
| InvariantSet::READ_AFTER_WRITE
| InvariantSet::RESTART_IDEMPOTENT,
limits: RunLimits {
max_wall_ms: Some(WallMs(60_000)),
},
restart_policy: RestartPolicy::EveryNOps(OpInterval(250)),
store: StoreConfig {
max_file_size: MaxFileSize(16 * 1024),
group_commit: GroupCommitConfig::default(),
shard_count: ShardCount(1),
},
eventlog: None,
writer_concurrency: WriterConcurrency(16),
};
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
assert_eq!(report.ops_executed.0, 1_000);
assert!(report.restarts.0 >= 2);
}
#[tokio::test]
async fn contended_writers_pristine_smoke() {
let cfg = GauntletConfig {
seed: Seed(2),
io: IoBackend::Simulated {
fault: FaultConfig::none(),
},
workload: WorkloadModel {
weights: OpWeights {
add: 85,
delete: 5,
compact: 3,
checkpoint: 2,
read_record: 4,
read_block: 1,
..OpWeights::default()
},
size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
key_space: KeySpaceSize(500),
did_space: DidSpaceSize(32),
retention_max_secs: RetentionMaxSecs(3600),
},
op_count: OpCount(1_000),
invariants: InvariantSet::REFCOUNT_CONSERVATION
| InvariantSet::REACHABILITY
| InvariantSet::ACKED_WRITE_PERSISTENCE
| InvariantSet::READ_AFTER_WRITE
| InvariantSet::RESTART_IDEMPOTENT,
limits: RunLimits {
max_wall_ms: Some(WallMs(60_000)),
},
restart_policy: RestartPolicy::EveryNOps(OpInterval(250)),
store: StoreConfig {
max_file_size: MaxFileSize(16 * 1024),
group_commit: GroupCommitConfig::default(),
shard_count: ShardCount(1),
},
eventlog: None,
writer_concurrency: WriterConcurrency(8),
};
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
assert_eq!(report.ops_executed.0, 1_000);
assert!(report.restarts.0 >= 2);
}
#[tokio::test]
async fn report_carries_generated_ops_when_clean() {
let cfg = fast_sanity_config(Seed(5));
let expected_len = cfg.op_count.0;
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert_clean(&report);
assert_eq!(
report.ops.len(),
expected_len,
"clean report missing op stream"
);
}
#[tokio::test]
async fn regression_round_trip_replays_injected_ops() {
let overrides = ConfigOverrides {
op_count: Some(25),
store: StoreOverrides {
max_file_size: Some(8192),
..StoreOverrides::default()
},
..ConfigOverrides::default()
};
let mut cfg = config_for(Scenario::SmokePR, Seed(99));
overrides.apply_to(&mut cfg);
let original_report = Gauntlet::new(cfg.clone())
.expect("build gauntlet")
.run()
.await;
let captured_ops = original_report.ops.clone();
assert_eq!(
captured_ops.len(),
25,
"captured op stream must match op_count override"
);
let dir = tempfile::TempDir::new().unwrap();
let record = RegressionRecord::from_report(
Scenario::SmokePR,
overrides.clone(),
&original_report,
captured_ops.len(),
captured_ops.clone(),
);
let written = record.write_to(dir.path()).expect("write regression");
let loaded = RegressionRecord::load(&written).expect("load regression");
assert_eq!(loaded.overrides, overrides);
assert_eq!(loaded.ops.len(), captured_ops.len());
let rebuilt = loaded.build_config().expect("rebuild config");
assert_eq!(rebuilt.op_count.0, 25);
assert_eq!(rebuilt.store.max_file_size.0, 8192);
let replay = Gauntlet::new(rebuilt)
.expect("build gauntlet")
.run_with_ops(loaded.op_stream())
.await;
assert_eq!(
replay.violations.len(),
original_report.violations.len(),
"replay from regression must produce same violation count",
);
let original_inv: Vec<&'static str> = original_report
.violations
.iter()
.map(|v| v.invariant)
.collect();
let replay_inv: Vec<&'static str> = replay.violations.iter().map(|v| v.invariant).collect();
assert_eq!(original_inv, replay_inv);
assert_eq!(replay.ops.len(), captured_ops.len());
}
#[tokio::test]
#[ignore = "long running, 100k ops with around 20 restarts"]
async fn mst_restart_churn_single_seed() {
let cfg = config_for(Scenario::MstRestartChurn, Seed(42));
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
assert!(
report.is_clean(),
"violations: {:?}",
report
.violations
.iter()
.map(|v| format!("{}: {}", v.invariant, v.detail))
.collect::<Vec<_>>()
);
assert!(
report.restarts.0 >= 1,
"PoissonByOps(5000) over 100k ops should fire at least 1 restart, got {}",
report.restarts.0
);
assert_clean(&report);
assert!(report.restarts.0 >= 1);
}

6
crates/tranquil-store/tests/proptests.rs
View File

@@ -2,8 +2,8 @@ use proptest::prelude::*;
use std::path::Path;
use tranquil_store::{
FaultConfig, HEADER_SIZE, OpenOptions, ReadRecord, RecordReader, RecordWriter, SimulatedIO,
StorageIO, run_crash_test, run_pristine_comparison, sim_proptest_cases,
FaultConfig, HEADER_SIZE, OpenOptions, Probability, ReadRecord, RecordReader, RecordWriter,
SimulatedIO, StorageIO, run_crash_test, run_pristine_comparison, sim_proptest_cases,
};
fn arb_payloads(max_count: usize, max_size: usize) -> BoxedStrategy<Vec<Vec<u8>>> {
@@ -151,7 +151,7 @@ proptest! {
data in proptest::collection::vec(any::<u8>(), 64..4096),
) {
let config = FaultConfig {
partial_write_probability: 0.5,
partial_write_probability: Probability::new(0.5),
..FaultConfig::none()
};
let dir = Path::new("/test");

23
crates/tranquil-store/tests/sim_eventlog.rs
View File

@@ -8,7 +8,7 @@ use tranquil_store::eventlog::{
DidHash, EVENT_RECORD_OVERHEAD, EventLogWriter, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD,
SEGMENT_HEADER_SIZE, SegmentId, SegmentManager, SegmentReader, ValidEvent,
};
use tranquil_store::{FaultConfig, SimulatedIO, StorageIO, sim_seed_range};
use tranquil_store::{FaultConfig, Probability, SimulatedIO, StorageIO, sim_seed_range};
use common::Rng;
@@ -529,8 +529,8 @@ fn group_sync_crash_after_append_before_sync() {
fn group_sync_crash_mid_sync_partial_fsync() {
sim_seed_range().into_par_iter().for_each(|seed| {
let fault_config = FaultConfig {
sync_failure_probability: 0.3,
partial_write_probability: 0.1,
sync_failure_probability: Probability::new(0.3),
partial_write_probability: Probability::new(0.1),
..FaultConfig::none()
};
let sim = SimulatedIO::new(seed, fault_config);
@@ -682,9 +682,9 @@ fn group_sync_no_double_sync_no_skipped_events() {
fn group_sync_contention_under_faults() {
sim_seed_range().into_par_iter().for_each(|seed| {
let fault_config = FaultConfig {
partial_write_probability: 0.05,
sync_failure_probability: 0.10,
dir_sync_failure_probability: 0.05,
partial_write_probability: Probability::new(0.05),
sync_failure_probability: Probability::new(0.10),
dir_sync_failure_probability: Probability::new(0.05),
..FaultConfig::none()
};
let sim = SimulatedIO::new(seed, fault_config);
@@ -897,12 +897,11 @@ fn multi_rotation_crash_at_each_phase() {
#[test]
fn aggressive_faults_group_sync_recovery() {
let fault_config = FaultConfig {
partial_write_probability: 0.15,
sync_failure_probability: 0.10,
dir_sync_failure_probability: 0.05,
misdirected_write_probability: 0.05,
bit_flip_on_read_probability: 0.0,
io_error_probability: 0.0,
partial_write_probability: Probability::new(0.15),
sync_failure_probability: Probability::new(0.10),
dir_sync_failure_probability: Probability::new(0.05),
misdirected_write_probability: Probability::new(0.05),
..FaultConfig::none()
};
sim_seed_range().into_par_iter().for_each(|seed| {