diff --git a/Cargo.lock b/Cargo.lock index 4596a2e11b..6ed7a82b1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12307,7 +12307,9 @@ dependencies = [ "server_common", "shard", "strum 0.28.0", + "tempfile", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index bdbcd2c2f8..8fba5f3bd5 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -85,7 +85,7 @@ pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_comma pub use error::WireError; pub use framing::{RequestFrame, ResponseFrame, STATUS_OK}; pub use primitives::ack_level::AckLevel; -pub use primitives::consumer::{KIND_CONSUMER_GROUP, WireConsumer}; +pub use primitives::consumer::{KIND_CONSUMER, KIND_CONSUMER_GROUP, WireConsumer}; pub use primitives::identifier::{MAX_WIRE_NAME_LENGTH, WireIdentifier, WireName}; pub use primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions, validate_options}; pub use primitives::partition_assignment::CreatedPartitionAssignment; diff --git a/core/binary_protocol/src/primitives/consumer.rs b/core/binary_protocol/src/primitives/consumer.rs index 202de84f18..83219c1c79 100644 --- a/core/binary_protocol/src/primitives/consumer.rs +++ b/core/binary_protocol/src/primitives/consumer.rs @@ -20,7 +20,8 @@ use crate::WireIdentifier; use crate::codec::{WireDecode, WireEncode, read_u8}; use bytes::{BufMut, BytesMut}; -const KIND_CONSUMER: u8 = 1; +/// Wire discriminant for a single consumer (vs a `ConsumerGroup`). +pub const KIND_CONSUMER: u8 = 1; /// Wire discriminant for a consumer-group consumer (vs a single `Consumer`). /// Public so the server dispatch can match on it by name instead of a raw `2`. pub const KIND_CONSUMER_GROUP: u8 = 2; diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 3af7ab0d7c..b7b7f83375 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -338,7 +338,10 @@ impl SnapshotCoordinator { /// forced. Must stay >= the prepare-queue depth: the ops already /// pipelined while a checkpoint runs skip it and append into this /// margin. - const CHECKPOINT_MARGIN: usize = 64; + /// + /// Public so a caller sizing a journal can refuse a slot count at or below it: + /// such a journal checkpoints on every commit rather than on occupancy. + pub const CHECKPOINT_MARGIN: usize = 64; #[must_use] pub fn new( @@ -366,7 +369,9 @@ impl SnapshotCoordinator { /// transfer serves and installs. #[must_use] pub fn snapshot_path(&self) -> std::path::PathBuf { - self.data_dir.join(super::METADATA_DIR).join("snapshot.bin") + self.data_dir + .join(super::METADATA_DIR) + .join(super::SNAPSHOT_FILE_NAME) } /// The last persisted checkpoint's `(op, checksum)`, `(0, 0)` when none. @@ -1170,21 +1175,48 @@ where // guard, not here. self.checkpoint_if_needed(consensus, journal).await; - // Backup: gap check (op == current_op + 1). - // Primary: sequencer pre-advanced by push_prepare_entry (guards - // sibling on_request races during journal.append await). - // TODO: promote the backup gap warn below to a hard assert or a - // repair-session trigger (message repair has landed; the drop-and- - // wait-for-retransmit path is the last soft handling left here). + // Backup: gap check against the JOURNAL head, not the sequencer. + // + // The two frontiers can disagree. The sequencer is pre-advanced on the + // primary by `push_prepare_entry` and re-synced on a backup only after a + // successful append, so a replica can carry a sequencer one ahead of what + // its WAL holds. Gating admission on it then rejects the very prepare that + // would heal the log: a backup with `last_op = 44` refused op 45 because + // its sequencer said to expect 46. The primary retransmits that op for the + // life of the process, every backup logs an out-of-order gap, it never + // reaches a commit quorum, and its client is never answered. + // + // `max(last_op, snapshot_op)`, never `last_op` alone. A state transfer + // installs a snapshot that IS ops `..=snapshot_op` applied and truncates the + // WAL above that floor rather than refilling below it, so `last_op` reads the + // receiver as needing an op the snapshot already contains and no peer will + // send again. That drop + // never heals: an offer built on a quiet cluster carries `commit_op == + // snapshot_seq`, so the install lands `commit_min == commit_max`, and + // `maybe_request_metadata_repair`, the only path that refills the head, + // arms on `commit_min < commit_max`. With the other backup down the primary + // needs this replica's ack to commit anything, so the plane stops on a + // cluster still inside its quorum. + // + // The journal is the only frontier that answers "what can be appended + // next", which is what this check is for, and the hash-chain verification + // below is stated against it too. A prepare at or below the head that this + // replica already holds was re-acked and returned above. What reaches HERE + // is the next op or a gap, and not every gap is fillable: metadata repair + // covers only `commit_min + 1 ..= commit_max`, so an interior hole below + // the head and a forward gap above `commit_max` both sit outside it. let is_backup = consensus.is_follower(); if is_backup { - if header.op != current_op + 1 { + let handle = journal.handle(); + let journal_head = handle.last_op().unwrap_or(0).max(handle.snapshot_op()); + if header.op != journal_head + 1 { warn!( target: "iggy.metadata.diag", plane = "metadata", replica_id = consensus.replica(), op = header.op, - expected = current_op + 1, + expected = journal_head + 1, + sequencer_op = current_op, "on_replicate: dropping out-of-order prepare (gap)" ); return; @@ -4837,6 +4869,201 @@ mod tests { ); } + /// A backup admits the prepare its JOURNAL needs next, even when its + /// sequencer has run ahead of the journal. + /// + /// The two frontiers legitimately disagree: `on_start_view` sets the sequencer + /// to the view's announced head, deliberately ahead of what this replica + /// holds, because the bodies arrive afterwards by retransmit or repair. Gating + /// admission on the sequencer therefore rejected exactly the prepare that + /// would heal the log: a backup with journal head 44 refusing op 45 because + /// its adopted head said to expect 46. The primary retransmits that op + /// forever, every backup logs an out-of-order gap, it never reaches a commit + /// quorum, and its client is never answered. Systematic for any rejoining + /// replica, so the deterministic simulator wedged on every metadata workload + /// under crash/restart injection until this was gated on the journal. + #[compio::test] + async fn backup_admits_the_prepare_its_journal_needs_despite_a_leading_sequencer() { + const CLIENT: u128 = 1; + const SESSION: u64 = 1; + const ACTING_USER: u32 = 7; + /// Stands in for a head adopted from a `StartView` whose bodies have not + /// arrived, so it sits well above the empty journal. + const ADOPTED_HEAD: u64 = 5; + + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join(crate::impls::METADATA_DIR)).unwrap(); + let journal = + journal::prepare_journal::PrepareJournal::open(&dir.path().join("journal.wal"), 0) + .await + .unwrap(); + // Replica 1 of 3 at view 0, so `primary_index(0) == 0` makes this a backup + // and `on_replicate` takes the gap-check branch. + let consensus = VsrConsensus::new( + 1, + 1, + 3, + server_common::sharding::METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + let md: IggyMetadata<_, journal::prepare_journal::PrepareJournal, (), TestMux> = + IggyMetadata::new( + Some(consensus), + Some(journal), + None, + None, + TestMux::default(), + Some(dir.path().to_path_buf()), + ); + let consensus = md.consensus.as_ref().unwrap(); + assert!( + consensus.is_follower(), + "replica 1 of 3 at view 0 must be a backup for this to exercise the gap check" + ); + md.client_table.borrow_mut().commit_register( + CLIENT, + ACTING_USER, + register_reply(CLIENT, SESSION), + ); + + // Minted while the sequencer is still at 0, so it carries op 1: exactly + // what the empty journal needs next. + let prepare = md + .prepare_request(create_stream_request(CLIENT, 1, "s1")) + .expect("CreateStream is client-allowed"); + assert_eq!(prepare.header().op, 1, "the first prepare must be op 1"); + + // Now run the sequencer ahead, as adopting a started view does. + consensus.sequencer().set_sequence(ADOPTED_HEAD); + let journal = md.journal.as_ref().unwrap(); + assert_eq!( + journal.last_op(), + None, + "the journal must still be empty, else the divergence under test is absent" + ); + + md.on_replicate(prepare).await; + + assert!( + journal.header(1).is_some(), + "backup dropped the prepare its journal needed next because its sequencer \ + was ahead; the primary's retransmit of this op can never be accepted, so \ + the op never commits and its client never gets a reply" + ); + } + + /// A state-transfer receiver admits the first live prepare above the floor it + /// installed, instead of waiting for an op the snapshot already contains. + /// + /// `install_state_transfer` moves the snapshot floor, the commit floor, the + /// sequencer and `commit_max`, and leaves the WAL head where it was: the + /// snapshot IS every op below the floor, so there is nothing left to append + /// for them. A gap check reading `last_op` alone therefore has the receiver + /// ask for `last_op + 1`, an op inside the snapshot that no peer will send + /// again, and it drops every live prepare forever. + /// + /// Nothing recovers it. An offer built on a quiet cluster carries `commit_op + /// == snapshot_seq`, so the install lands `commit_min == commit_max` and + /// `maybe_request_metadata_repair`, gated on `commit_min < commit_max`, never + /// arms; repair is the only path that could refill the head. With the + /// other backup down the primary needs this replica's ack to commit at all, + /// so a 3-node cluster still inside its quorum stops serving metadata. + #[compio::test] + async fn state_transfer_receiver_admits_the_first_prepare_above_the_installed_floor() { + const CLIENT: u128 = 1; + const SESSION: u64 = 1; + const ACTING_USER: u32 = 7; + /// The `snapshot_seq` of a transferred offer, far above anything this + /// replica's own WAL holds. + const INSTALLED_FLOOR: u64 = 400; + + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join(crate::impls::METADATA_DIR)).unwrap(); + let journal = + journal::prepare_journal::PrepareJournal::open(&dir.path().join("journal.wal"), 0) + .await + .unwrap(); + // Replica 1 of 3 at view 0, so `primary_index(0) == 0` makes this a backup + // and `on_replicate` takes the gap-check branch. + let consensus = VsrConsensus::new( + 1, + 1, + 3, + server_common::sharding::METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + let md: IggyMetadata<_, journal::prepare_journal::PrepareJournal, (), TestMux> = + IggyMetadata::new( + Some(consensus), + Some(journal), + None, + None, + TestMux::default(), + Some(dir.path().to_path_buf()), + ); + let consensus = md.consensus.as_ref().unwrap(); + assert!( + consensus.is_follower(), + "replica 1 of 3 at view 0 must be a backup for this to exercise the gap check" + ); + md.client_table.borrow_mut().commit_register( + CLIENT, + ACTING_USER, + register_reply(CLIENT, SESSION), + ); + + // Give the WAL a head far below the floor about to be installed, which is + // what a replica that fell behind its peers' retention actually carries. + let first = md + .prepare_request(create_stream_request(CLIENT, 1, "s1")) + .expect("CreateStream is client-allowed"); + md.on_replicate(first).await; + let journal = md.journal.as_ref().unwrap(); + assert_eq!( + journal.last_op(), + Some(1), + "the WAL head must sit below the installed floor, else the divergence \ + under test is absent" + ); + + // Exactly the frontiers `install_state_transfer` leaves for a quiet-cluster + // offer, where the manifest's `commit_op` equals its `snapshot_seq`. + journal.set_snapshot_op(INSTALLED_FLOOR); + consensus.set_commit_floor(INSTALLED_FLOOR); + consensus.sequencer().set_sequence(INSTALLED_FLOOR); + consensus.advance_commit_max(INSTALLED_FLOOR); + assert_eq!( + consensus.commit_min(), + consensus.commit_max(), + "the wedge needs an install with no repair left to arm; diverged \ + frontiers heal through `maybe_request_metadata_repair`" + ); + + let next = md + .prepare_request(create_stream_request(CLIENT, 2, "s2")) + .expect("CreateStream is client-allowed"); + assert_eq!( + next.header().op, + INSTALLED_FLOOR + 1, + "the primary numbers the next op off the installed floor" + ); + + md.on_replicate(next).await; + + assert!( + journal + .header(usize::try_from(INSTALLED_FLOOR + 1).unwrap()) + .is_some(), + "state-transfer receiver dropped the first prepare above its installed \ + floor; the ops the snapshot already holds are never re-sent, and with \ + `commit_min == commit_max` no repair arms, so this op never commits" + ); + } + /// A checkpoint reclaims the WAL prefix the snapshot supersedes, but must /// stop one op short of the checkpoint op itself. /// diff --git a/core/metadata/src/impls/mod.rs b/core/metadata/src/impls/mod.rs index 8c9c2dca07..55dbf855f7 100644 --- a/core/metadata/src/impls/mod.rs +++ b/core/metadata/src/impls/mod.rs @@ -20,3 +20,8 @@ pub mod recovery; /// Subdirectory under the data root where metadata state is stored. pub const METADATA_DIR: &str = "metadata"; + +/// File under [`METADATA_DIR`] holding the persisted snapshot, and the artifact +/// state transfer serves. Public so a reader outside this crate (the simulator +/// asserting a checkpoint landed) can name it. +pub const SNAPSHOT_FILE_NAME: &str = "snapshot.bin"; diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index fb1ba35c2f..d7413752c4 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -779,6 +779,65 @@ where self.should_increment_offset = true; } + /// Whether this partition ever stamped an offset, i.e. whether its offset + /// counters describe a real offset space rather than an untouched zero. The + /// one bit separating a partition holding one message at offset 0 from one + /// that never took a write: both report `(0, 0)`. + #[cfg(any(test, feature = "simulator"))] + pub const fn offset_space_used(&self) -> bool { + self.should_increment_offset + } + + /// Adopt a log carried over from a previous incarnation of this partition, + /// standing in for what segment recovery reads off disk at boot. + /// + /// A real server loses nothing across the rebuild: its messages are in segment + /// files and boot recovers the offset counter from them. The simulator's + /// partitions are in-memory, so without this the rebuilt partition comes back + /// empty and its `commit_offset` regresses to zero, which reads as a consensus + /// regression rather than the harness having thrown the data away. + /// + /// `durable_offset` and `write_offset` are what the caller recovered, as + /// `segment_recovery` derives them from segments. Applied as a MAX against + /// whatever the superblock frontier already proved, for the same reason + /// [`Self::restore_offset_frontier`] maxes: a recovered value behind the + /// frontier must not lower it. + #[cfg(any(test, feature = "simulator"))] + pub fn adopt_retained_log(&mut self, state: crate::RetainedPartitionState) { + let crate::RetainedPartitionState { + log, + durable_offset, + write_offset, + offset_space_used, + } = state; + self.log = log; + // Empty carry-over: the previous incarnation never took a write, so there + // is no offset space to restore and claiming one would make the next + // prepare mint from a base no peer agrees on. + // + // Keyed on the RETIRED incarnation's flag, never on `(0, 0)` or on this + // partition's own `should_increment_offset`. One message at offset 0 reports + // the same two zeroes as an empty partition, and this instance is freshly + // built so its own flag is always false. The arithmetic test would therefore + // adopt the log, skip the counters, and let the next write stamp + // `base_offset = 0` where peers stamp 1, with `batch_checksum` over it: two + // logs, different bytes at the same op, silently. + if !offset_space_used { + return; + } + let durable = durable_offset.max(self.offset.load(Ordering::Acquire)); + let dirty = write_offset + .max(durable) + .max(self.dirty_offset.load(Ordering::Relaxed)); + self.offset.store(durable, Ordering::Release); + self.dirty_offset.store(dirty, Ordering::Relaxed); + self.should_increment_offset = true; + // Everything carried over is already persisted as far as this replica is + // concerned, so the flush and commit paths must not re-persist or re-count + // it, the same contract boot gives a partition recovered from segments. + self.recovered_durable_offset = Some(durable); + } + /// Copy this incarnation's offset counter into the shared /// [`PartitionStats`], making it the value readers (offset validation, /// `get_topic`, `get_stats`) see. @@ -2269,7 +2328,16 @@ where } // Backup gap check; primary sequencer pre-advanced by - // push_prepare_entry. See metadata::on_replicate. + // push_prepare_entry. + // + // The sequencer, deliberately, where `metadata::on_replicate` gates on its + // journal. The two frontiers cannot drift apart on this plane: + // `install_state_transfer` rewinds the sequencer to the offer's `commit_op` + // (where the metadata install moves a durable snapshot floor and leaves the + // WAL head behind it), and the repair ingest advances it by walking the + // journal. Reading the journal here would answer 0 after every restart, + // since this plane's journal is memory-only and starts empty however much + // data sits on disk. let is_backup = self.consensus().is_follower(); if is_backup { if header.op != current_op + 1 { diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index ac5f1909fa..ec45aefb9f 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -824,6 +824,23 @@ where } } + /// Highest `commit` any resident header stamped, in ONE pass. + /// + /// A lower bound on the group's commit point, which is what a rebuilt replica + /// can recover from a log alone: a prepare records the primary's commit point + /// at send time, so the true point may be one higher. + /// + /// Exists so callers do not walk `1..=head` through [`Self::header_by_op`], + /// which is a linear scan per op and so quadratic in the head. + pub fn max_commit_watermark(&self) -> u64 { + let headers = unsafe { &*self.headers.get() }; + headers + .iter() + .map(|header| header.commit) + .max() + .unwrap_or(0) + } + /// Headers for the contiguous op run `from_op ..= commit_max`, in op order, /// stopping at the first missing op. A replication gap must not be skipped: /// the caller advances `commit_min` strictly by one, so a hole would break diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 4649af8290..aa33f5c7a8 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -51,6 +51,30 @@ pub use types::{ RepairSession, SendMessagesResult, }; +/// A partition's message log, named so a caller can carry one across a rebuild. +/// +/// Exists for the simulator, which has no segment files and so must hold the log +/// itself for a restarted replica to come back with its data (see +/// [`IggyPartition::adopt_retained_log`]). Names the only journal +/// `IggyPartition::log` is instantiated with rather than widening anything. +#[cfg(any(test, feature = "simulator"))] +pub type RetainedPartitionLog = + log::SegmentedLog>; + +/// Everything a partition hands its own next incarnation across a simulated +/// restart. +#[cfg(any(test, feature = "simulator"))] +pub struct RetainedPartitionState { + pub log: RetainedPartitionLog, + /// Offset counter the previous incarnation had proved durable. + pub durable_offset: u64, + /// Highest offset it had written, durable or not. + pub write_offset: u64, + /// Whether that incarnation ever stamped an offset, i.e. whether the two + /// numbers above describe an offset space at all. + pub offset_space_used: bool, +} + /// Partition-level data plane operations. /// /// `send_messages` MUST only append to the partition journal (prepare phase), diff --git a/core/shard/Cargo.toml b/core/shard/Cargo.toml index b803be13a5..96a72c9117 100644 --- a/core/shard/Cargo.toml +++ b/core/shard/Cargo.toml @@ -28,7 +28,9 @@ publish = false # off the pump task. A `-p iggy-server` build excludes it; `cargo build # --workspace` unifies features and compiles it into the shared `shard` # unit (simulator requests it). Benign: no production caller. -simulator = [] +# Forwards to `partitions/simulator`: `init_partition` names +# `RetainedPartitionLog` and calls `adopt_retained_log`, both gated there. +simulator = ["partitions/simulator"] [dependencies] compio = { workspace = true } diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 1efaa598a3..ab78033a0a 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -25,7 +25,7 @@ pub mod shards_table; pub use config::CoordinatorConfig; pub use router::CONSENSUS_TICK_INTERVAL; -#[cfg(any(test, feature = "simulator"))] +#[cfg(feature = "simulator")] use consensus::LocalPipeline; use consensus::{ ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, DvcHeaderKind, @@ -46,7 +46,7 @@ use iggy_binary_protocol::{ RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, }; -#[cfg(any(test, feature = "simulator"))] +#[cfg(feature = "simulator")] use iggy_common::PartitionStats; use iggy_common::variadic; use iggy_common::{IggyError, IggyExpiry, IggyTimestamp}; @@ -70,7 +70,7 @@ use std::cell::{Cell, RefCell}; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::future::Future; use std::rc::Rc; -#[cfg(any(test, feature = "simulator"))] +#[cfg(feature = "simulator")] use std::sync::Arc; pub type ShardPlane = @@ -3518,32 +3518,39 @@ where total } - /// Simulator-only. Mutates `IggyPartitions` off the pump task, - /// bypassing the reconciler's `ReconcileOp::InsertOwned` funnel (the - /// production runtime path; bootstrap recovery uses `load_partition`), - /// so it must never run in production. VSR replica id comes from - /// `PartitionConsensusConfig`, not `self.id` (the local shard index). A - /// `-p iggy-server` build excludes the `simulator` feature and this - /// method; `cargo build --workspace` compiles it in but with no - /// production caller. - /// `superblock` is this group's durable `(view, log_view)` store. Passing - /// `None` keeps the storeless branch, where the persist gate marks every view - /// durable without writing anything -- fine for specs that never restart a - /// replica, but it means the gate itself, its write-failure fence, and view - /// recovery are all unexercised. A caller that hands one in (the simulator, - /// which retains the store across a replica rebuild) gets the production - /// contract: a recorded view is restored before the group joins, and a failed - /// write withholds every view-scoped send. + /// Simulator-only: mutates `IggyPartitions` off the pump task, bypassing the + /// reconciler's `ReconcileOp::InsertOwned` funnel (production's runtime path; + /// bootstrap recovery uses `load_partition`). VSR replica id comes from + /// `PartitionConsensusConfig`, not `self.id` (the local shard index). /// - /// `recovered_state` is that store's last record, read by the caller (the - /// store's read is async and this is not), mirroring how `new_shard` takes the - /// metadata plane's. - #[cfg(any(test, feature = "simulator"))] + /// `superblock` is this group's durable `(view, log_view)` store. `None` takes + /// the storeless branch, where the persist gate marks every view durable + /// without writing, leaving the gate, its write-failure fence and view + /// recovery unexercised. Passing one in gets the production contract: a + /// recorded view is restored before the group joins, and a failed write + /// withholds every view-scoped send. `recovered_state` is that store's last + /// record, read by the caller because the store's read is async and this is + /// not. + /// + /// `retained` is the log a previous incarnation left behind, standing in for + /// the segments a real boot recovers from. `None` is right for a first + /// materialisation and wrong for a restart: a rebuilt partition with no data + /// reports `commit_offset` 0, which reads as a regression rather than a + /// harness that discarded the log. + // `feature = "simulator"` alone, unlike its neighbours: the body names items + // `partitions` gates the same way, and a `test` arm cannot turn those on. + // Under `cargo test -p shard` that arm fires from shard's own `cfg(test)` + // while `partitions` builds as a plain dependency, so `RetainedPartitionLog` + // and `adopt_retained_log` are configured out and the crate does not compile. + // The feature forwards to `partitions/simulator` instead. + #[cfg(feature = "simulator")] pub fn init_partition( &self, namespace: IggyNamespace, superblock: Option>, recovered_state: Option, + retained: Option, + restore_frontier: bool, ) where B: MessageBus + Clone, { @@ -3568,7 +3575,20 @@ where consensus.set_log_view(state.log_view); consensus.mark_superblock_durable(state.view, state.log_view); } - consensus.init(); + // Boot as `load_partition` does. A rebuilt replica cannot know the group's + // `(op, commit)`: the partition journal is in-memory and segments carry no + // op numbers. So in a cluster it joins quorum-invisible and asks the view's + // primary rather than resuming as a primary its peers may have replaced. + // Plain `init` would set `Status::Normal` and arm the commit broadcast on + // whichever replica is primary-by-index, the split-brain `init_as_backup` + // exists to prevent. A first materialisation has no view to rejoin and + // keeps the plain init; `retained` is populated only by the restart path. + if retained.is_some() && self.partition_consensus.replica_count > 1 { + consensus.init_as_backup(); + consensus.begin_view_probe(); + } else { + consensus.init(); + } let stats = Arc::new(PartitionStats::default()); let mut partition = IggyPartition::with_in_memory_storage( @@ -3580,6 +3600,39 @@ where if let Some(superblock) = superblock { partition.set_superblock(superblock, recovered_state.as_ref()); } + // Retained log before the frontier restore, so the restore maxes against + // the offsets the log proved rather than the zeroes of an empty one. + // `restore_offset_frontier` STORES `recovered_end` once past its guard, so + // it can lower `dirty_offset`; harmless only because `write_superblock` + // maxes the recorded frontier against `offset_frontier()`. The order also + // keeps that restore's precondition (`should_increment_offset` already set + // by a recovered offset space) meaningful. + if let Some(state) = retained { + partition.adopt_retained_log(state); + // OPT-IN, off by default: it models durability Iggy does not have. + // Production's `load_partition` restores the view alone, joins as a + // backup and probes, so a harness handing the frontier back cannot + // reproduce the empty-frontier restart that is the real hazard. With it + // off a restarted replica rebuilds at op 0 while holding a log full of + // ops and ADVERTISES that empty frontier in its `DoViewChange`, which + // trips the sequential-advance assert in `advance_commit_min`. A + // scenario turns this on only to look past that at something later in + // the run. + // + // `max_commit_watermark` is a lower bound: a prepare records the + // primary's commit point at send time, so the true point may be one + // higher and re-commits on rejoin. + let journal = &partition.log.journal().inner; + if restore_frontier && let Some(head) = journal.last_op() { + let watermark = journal.max_commit_watermark(); + let consensus = partition.consensus(); + consensus.sequencer().set_sequence(head); + consensus.restore_commit_state(watermark, watermark); + if let Some(header) = journal.header_by_op(head) { + consensus.set_last_prepare_checksum(header.checksum); + } + } + } // The SAME call the boot paths make, not a copy of it: this restore is // a max against what the segments already proved, and a harness running // a divergent copy of that rule cannot catch a violation of it. Without diff --git a/core/simulator/Cargo.toml b/core/simulator/Cargo.toml index 4f837738ed..d24918b527 100644 --- a/core/simulator/Cargo.toml +++ b/core/simulator/Cargo.toml @@ -48,6 +48,10 @@ server_common = { path = "../server_common", features = ["simulator"] } shard = { path = "../shard", features = ["simulator"] } strum = { workspace = true } tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } [lints.clippy] enum_glob_use = "deny" diff --git a/core/simulator/src/bin/workload-fuzz.rs b/core/simulator/src/bin/workload-fuzz.rs index 11d8c865f5..0bde63a00d 100644 --- a/core/simulator/src/bin/workload-fuzz.rs +++ b/core/simulator/src/bin/workload-fuzz.rs @@ -17,38 +17,49 @@ //! Deterministic workload fuzzer for the Iggy simulator. //! -//! Drives [`simulator::workload::run`] (per-tick invariants + optional crash -//! injection) for a number of ticks, then optionally quiesces and asserts the -//! Phase C consensus checks. Everything is a function of `--seed`, logged at -//! start and on panic so any failure replays with `--seed `. +//! Drives [`simulator::workload::run_with_faults`] (per-tick invariants plus +//! crash, restart and network fault injection) for a number of ticks, then +//! optionally quiesces and asserts the Phase C consensus checks. Everything is a +//! function of `--seed`, logged at start and on panic so any failure replays +//! with `--seed `. //! //! ```text //! workload-fuzz [--seed N] [--ticks N] [--clients N] [--replicas N] -//! [--crash-prob F] [--no-quiesce] +//! [--plane partition|metadata|mixed|uniform] +//! [--faults none|light|heavy|swarm] [--no-quiesce] +//! [--crash-prob F] [--restart-prob F] [--crash-primary] +//! [network overrides: --packet-loss-prob, --replay-prob, --partition-mode, +//! --partition-prob, --unpartition-prob, --clog-prob, ...] //! ``` //! -//! The default workload is partition-plane (`SendMessages`): it drains and -//! converges. Metadata and mixed-plane workloads are gated on the metadata -//! request-gap: a client's replicated-metadata request ids must arrive -//! contiguously (`committed + 1`), so a dropped or reordered metadata request -//! opens a permanent `RequestGap` that wedges that client's metadata plane. -//! Broader op coverage lands once the workload generator models that -//! constraint. - -use clap::Parser; +//! `--plane` selects the op mix (see [`ActionWeights`]). Partition-plane runs drain +//! and converge most readily; `uniform` is the widest per-tick op coverage. +//! +//! `--faults` picks a whole network fault profile, and the individual network flags +//! override single fields of it, so exploring one axis does not mean spelling out +//! the other ten. The default is `none`, a perfect network, so a run says what it +//! injects rather than inheriting it. +//! +//! `--faults swarm` derives every network parameter from `--seed`, which is what a +//! CI campaign wants: `none`/`light`/`heavy` are three points in parameter space, +//! so a thousand seeds against `heavy` is the same network a thousand times. The +//! drawn values print on the `network:` line and `--seed` replays them exactly. + +use clap::{Parser, ValueEnum}; use iggy_common::IggyByteSize; use server_common::sharding::IggyNamespace; use server_common::{MemoryPool, MemoryPoolSettings}; use simulator::Simulator; use simulator::client::SimClient; -use simulator::packet::PacketSimulatorOptions; +use simulator::packet::{COMMAND_LABELS, PacketSimulatorOptions, PartitionMode, PartitionSymmetry}; use simulator::workload::actions::Action; use simulator::workload::options::{ActionWeights, WorkloadOptions}; -use simulator::workload::{Workload, oracle, run}; +use simulator::workload::{FaultInjector, Workload, oracle, run_with_faults}; use strum::IntoEnumIterator; #[derive(Parser)] #[command(about = "Deterministic workload fuzzer for the Iggy simulator")] +#[allow(clippy::struct_excessive_bools)] struct Args { /// Omitted draws a random seed (logged for replay). #[arg(long)] @@ -59,10 +70,282 @@ struct Args { clients: u8, #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u8).range(1..))] replicas: u8, + /// Op mix to draw from. + #[arg(long, value_enum, default_value_t = Plane::Partition)] + plane: Plane, + /// Probability a consumer-offset store asks for `Quorum` rather than + /// `NoAck`. `1.0` keeps every offset op on the replicated path. + #[arg(long, default_value_t = 0.5, value_parser = parse_unit_interval)] + ack_quorum_ratio: f32, + /// Per-tick chance one eligible replica is crashed. #[arg(long, default_value_t = 0.0, value_parser = parse_unit_interval)] crash_prob: f32, + /// Per-tick chance one crashed replica is restarted. Without this a crash + /// is permanent and nothing exercises rejoin or log repair. + #[arg(long, default_value_t = 0.0, value_parser = parse_unit_interval)] + restart_prob: f32, + /// Crash the primary too, putting a view change under live traffic. + #[arg(long)] + crash_primary: bool, + /// Bound every replica's metadata WAL to this many slots, which is what forces a + /// checkpoint (`should_checkpoint` gates on remaining capacity). Unbounded by + /// default, and an unbounded journal never checkpoints, so WAL drain, + /// `snapshot_op` movement, `RangeEvicted` and metadata state transfer are all + /// unreachable until this is set. + #[arg(long, value_parser = parse_journal_slots)] + journal_slots: Option, + /// Directory the checkpointing run writes its snapshots into, retained for + /// diagnosis. Defaults to a fresh per-process directory, whose path is printed. + /// Refuses an existing one unless `--reuse-data-dir` says so. + #[arg(long)] + data_dir: Option, + /// Allow `--data-dir` to name a directory that already exists. + #[arg(long)] + reuse_data_dir: bool, + /// Live replicas the fault injector will not crash below. Defaults to a commit + /// quorum (`replicas / 2 + 1`). `--replicas 1 --min-survivors 0` is the only way + /// to exercise a single-replica restart. + #[arg(long)] + min_survivors: Option, + /// Let a rebuilt partition recover its consensus frontier from the log the + /// harness carried across the restart. + /// + /// OFF by default, because a real replica cannot do this: the partition journal + /// is in-memory and segments carry no op numbers, so production's + /// `load_partition` restores the view alone and rejoins quorum-invisible. With it + /// off a restarted replica comes back at op 0 and the run exercises that rejoin, + /// where `advance_commit_min`'s sequential-advance assert lives. Turn it on to + /// look PAST that at something later in the run, studying a system more durable + /// than Iggy is. + #[arg(long)] + restore_partition_frontier: bool, + /// Fail the run if the entity oracle did not hold at quiesce. + /// + /// An eviction disarms it (the forgotten request's fate is unknown) and it + /// re-arms only once the shadow is proven equal to committed state again. Without + /// this flag a run whose oracle stayed disarmed still exits 0. + #[arg(long)] + require_entity_oracle: bool, + /// Committed workload operations this run must produce, or it fails. + /// + /// A run that commits nothing proved nothing: every oracle downstream compares an + /// empty shadow against empty committed state and agrees. `0` opts out. + #[arg(long, default_value_t = 1)] + min_commits: u64, + /// Committed metadata ops that must have been witnessed by more than one live + /// replica, i.e. that exercised cross-replica agreement. Ignored below two live + /// replicas, where the property is untestable rather than untested. `0` opts out. + #[arg(long, default_value_t = 1)] + min_ops_compared: usize, + /// Fail the run if crash or restart injection was requested but never happened. + /// Off by default, since a short run at low probability may legitimately draw + /// none; on for a campaign where such a seed is silently wasted. + #[arg(long)] + require_faults: bool, + /// Route every client request through the server's real dispatch handlers + /// instead of the raw `on_message` fast path. Clients then log in against the + /// seeded root user and carry a bound session, so the run also covers + /// authorization and session lifecycle, which exist only on this path. + #[arg(long)] + shell: bool, #[arg(long)] no_quiesce: bool, + + /// Network fault profile. Individual network flags below override single + /// fields of the profile. + #[arg(long, value_enum, default_value_t = Faults::None)] + faults: Faults, + /// Chance a packet is dropped at delivery time. + #[arg(long, value_parser = parse_unit_interval_f64)] + packet_loss_prob: Option, + /// Chance a packet is duplicated at delivery time. + #[arg(long, value_parser = parse_unit_interval_f64)] + replay_prob: Option, + /// Minimum one-way delay, in ticks. + #[arg(long)] + one_way_delay_min: Option, + /// Mean one-way delay, in ticks (exponentially distributed). + #[arg(long)] + one_way_delay_mean: Option, + /// Maximum packets queued on a single link; beyond it the link drops. + #[arg(long, value_parser = clap::value_parser!(u8).range(1..))] + link_capacity: Option, + /// How an automatic partition picks its sides. + #[arg(long, value_enum)] + partition_mode: Option, + /// Whether a partition blocks both directions or just one. + #[arg(long, value_enum)] + partition_symmetry: Option, + /// Per-tick chance a partition forms while connectivity is whole. + #[arg(long, value_parser = parse_unit_interval_f64)] + partition_prob: Option, + /// Per-tick chance a standing partition heals. + #[arg(long, value_parser = parse_unit_interval_f64)] + unpartition_prob: Option, + /// Minimum ticks a partition lasts once formed. + #[arg(long)] + partition_stability: Option, + /// Minimum ticks of whole connectivity before another partition may form. + #[arg(long)] + unpartition_stability: Option, + /// Per-tick chance any one path clogs (stops delivering, keeps queueing). + #[arg(long, value_parser = parse_unit_interval_f64)] + clog_prob: Option, + /// Mean clog duration, in ticks (exponentially distributed). + #[arg(long)] + clog_duration_mean: Option, +} + +/// Named network fault profile: one flag for "how hostile is the network", rather +/// than eleven. +/// +/// Progress falls off steeply with severity, every lost frame costing a resend +/// timeout: on one namespace with one client, a 3-replica cluster drains roughly 440 +/// replies in 5000 ticks on a perfect network, 240 under `light` and 40 under +/// `heavy`. All still drain and converge, so budget ticks accordingly rather than +/// reading a low reply count as a stall. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum Faults { + /// Perfect network. Delays only, no loss and no partitions. + None, + /// Occasional loss, duplication and short one-sided partitions. Meant to + /// stay inside the range where a healthy cluster still drains. + Light, + /// Frequent loss, long partitions and clogged paths. Progress stalls for + /// stretches, so budget ticks generously; the stalls are transient and a healthy + /// cluster still drains and converges, which is why the quiesce assert treats a + /// failure to drain as real rather than as expected weather. + Heavy, + /// Every parameter drawn from the seed. What a CI campaign should run: the three + /// fixed profiles above are three points in an eleven-dimensional space, so + /// throwing seeds at one of them varies the traffic and never the network. + /// Severity ranges over roughly `none` through half again `heavy`, so some seeds + /// draw a calm network and some worse than `heavy`; both are the point. See + /// [`PacketSimulatorOptions::swarm`]. + Swarm, +} + +/// Clap mirror of [`PartitionMode`], so the library type stays free of a clap +/// derive. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum PartitionModeArg { + None, + UniformSize, + UniformPartition, + IsolateSingle, +} + +impl From for PartitionMode { + fn from(value: PartitionModeArg) -> Self { + match value { + PartitionModeArg::None => Self::None, + PartitionModeArg::UniformSize => Self::UniformSize, + PartitionModeArg::UniformPartition => Self::UniformPartition, + PartitionModeArg::IsolateSingle => Self::IsolateSingle, + } + } +} + +/// Clap mirror of [`PartitionSymmetry`]; see [`PartitionModeArg`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum PartitionSymmetryArg { + Symmetric, + Asymmetric, +} + +impl From for PartitionSymmetry { + fn from(value: PartitionSymmetryArg) -> Self { + match value { + PartitionSymmetryArg::Symmetric => Self::Symmetric, + PartitionSymmetryArg::Asymmetric => Self::Asymmetric, + } + } +} + +impl Faults { + /// Base network options for this profile. `node_count`, `client_count` and + /// `seed` are filled by the caller. Takes the seed because [`Faults::Swarm`] + /// derives its whole shape from it; the fixed profiles ignore it, which is what + /// makes them reproducible without one. + fn options(self, seed: u64) -> PacketSimulatorOptions { + match self { + // `PacketSimulatorOptions::default` is already a perfect network: + // delay only, every probability zero. + Self::None => PacketSimulatorOptions::default(), + Self::Swarm => PacketSimulatorOptions::swarm(seed), + Self::Light => PacketSimulatorOptions { + packet_loss_probability: 0.02, + replay_probability: 0.01, + partition_probability: 0.005, + unpartition_probability: 0.05, + partition_stability: 20, + unpartition_stability: 40, + partition_mode: PartitionMode::IsolateSingle, + partition_symmetry: PartitionSymmetry::Asymmetric, + path_clog_probability: 0.002, + path_clog_duration_mean: 10, + ..PacketSimulatorOptions::default() + }, + Self::Heavy => PacketSimulatorOptions { + packet_loss_probability: 0.10, + replay_probability: 0.03, + one_way_delay_mean: 8, + partition_probability: 0.02, + unpartition_probability: 0.02, + partition_stability: 50, + unpartition_stability: 50, + partition_mode: PartitionMode::UniformSize, + partition_symmetry: PartitionSymmetry::Asymmetric, + path_clog_probability: 0.01, + path_clog_duration_mean: 25, + ..PacketSimulatorOptions::default() + }, + } + } +} + +/// Which plane the sampled ops target. Maps onto an [`ActionWeights`] preset. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum Plane { + /// Writes and consumer offsets only. + Partition, + /// Replicated metadata mutations only. + Metadata, + /// Stream creates over a write-heavy base. + Mixed, + /// Every action equally likely. + Uniform, +} + +impl Plane { + fn weights(self) -> ActionWeights { + match self { + Self::Partition => ActionWeights::partition_only(), + Self::Metadata => ActionWeights::metadata_only(), + Self::Mixed => ActionWeights::default(), + Self::Uniform => ActionWeights::uniform(), + } + } +} + +/// Clap value parser: a journal slot count a checkpoint can actually be driven by. +/// +/// At or below the coordinator's margin the journal sits at the threshold from the +/// first op, so the run checkpoints on every commit and measures that, not the +/// workload. +fn parse_journal_slots(raw: &str) -> Result { + let value: usize = raw + .parse() + .map_err(|_| format!("`{raw}` is not a whole number"))?; + let margin = metadata::impls::metadata::SnapshotCoordinator::<()>::CHECKPOINT_MARGIN; + if value > margin { + Ok(value) + } else { + Err(format!( + "must exceed the checkpoint margin ({margin}), or every commit checkpoints; \ + got {value}" + )) + } } /// Clap value parser: accept a probability in `[0.0, 1.0]`. @@ -77,28 +360,180 @@ fn parse_unit_interval(raw: &str) -> Result { } } +/// [`parse_unit_interval`] for the network knobs, which are `f64`. +fn parse_unit_interval_f64(raw: &str) -> Result { + let value: f64 = raw + .parse() + .map_err(|_| format!("`{raw}` is not a number"))?; + if (0.0..=1.0).contains(&value) { + Ok(value) + } else { + Err(format!("must be within [0.0, 1.0], got {value}")) + } +} + +/// What this run is, on two lines: the cluster and workload shape, then every +/// network parameter. +/// +/// Every network field, not the interesting subset. Under [`Faults::Swarm`] these +/// ARE the run's identity, and a report naming half of them cannot be read against +/// a failure without re-deriving the rest by hand. +fn print_run_banner(args: &Args, seed: u64, network: &PacketSimulatorOptions) { + println!( + "workload-fuzz: seed={seed} ticks={} clients={} replicas={} plane={:?} \ + faults={:?} shell={} crash_prob={} quiesce={}", + args.ticks, + args.clients, + args.replicas, + args.plane, + args.faults, + args.shell, + args.crash_prob, + !args.no_quiesce, + ); + println!( + "network: loss={} replay={} delay={}..{} partition={:?}/{:?} \ + p_partition={} p_unpartition={} stability={}/{} clog={} clog_ticks={} \ + link_capacity={}", + network.packet_loss_probability, + network.replay_probability, + network.one_way_delay_min, + network.one_way_delay_mean, + network.partition_mode, + network.partition_symmetry, + network.partition_probability, + network.unpartition_probability, + network.partition_stability, + network.unpartition_stability, + network.path_clog_probability, + network.path_clog_duration_mean, + network.link_capacity, + ); +} + +/// The chosen fault profile with any individually-set network flag applied over +/// it, plus the cluster shape and seed. +fn network_options(args: &Args, replicas: u8, clients: u8, seed: u64) -> PacketSimulatorOptions { + let mut options = args.faults.options(seed); + options.node_count = replicas; + options.client_count = clients; + options.seed = seed; + + if let Some(value) = args.packet_loss_prob { + options.packet_loss_probability = value; + } + if let Some(value) = args.replay_prob { + options.replay_probability = value; + } + if let Some(value) = args.one_way_delay_min { + options.one_way_delay_min = value; + } + if let Some(value) = args.one_way_delay_mean { + options.one_way_delay_mean = value; + } + if let Some(value) = args.link_capacity { + options.link_capacity = value; + } + if let Some(value) = args.partition_mode { + options.partition_mode = value.into(); + } + if let Some(value) = args.partition_symmetry { + options.partition_symmetry = value.into(); + } + if let Some(value) = args.partition_prob { + options.partition_probability = value; + } + if let Some(value) = args.unpartition_prob { + options.unpartition_probability = value; + } + if let Some(value) = args.partition_stability { + options.partition_stability = value; + } + if let Some(value) = args.unpartition_stability { + options.unpartition_stability = value; + } + if let Some(value) = args.clog_prob { + options.path_clog_probability = value; + } + if let Some(value) = args.clog_duration_mean { + options.path_clog_duration_mean = value; + } + + // Every other override above is one self-sufficient field. The partition knobs + // are not: the probability roll calls `auto_partition_network`, whose + // `PartitionMode::None` arm clears every side, so `--partition-prob 0.3` against + // the default `none` profile prints `p_partition=0.3 partition=None`, reports OK, + // and partitions zero times. Imply a mode when the caller asked for partitions + // without naming one; an explicit `--partition-mode` still wins. + let asked_for_partitions = args.partition_prob.is_some_and(|value| value > 0.0) + || args.unpartition_prob.is_some() + || args.partition_stability.is_some() + || args.unpartition_stability.is_some(); + if asked_for_partitions + && args.partition_mode.is_none() + && options.partition_mode == PartitionMode::None + { + options.partition_mode = PartitionMode::UniformSize; + } + options +} + +/// Report a network configuration that cannot do what it says, before the simulator +/// asserts on it several frames deeper. +/// +/// The delay pair is the one relationship no single parser can check: the two values +/// may arrive from different sources, a profile supplying one and a flag the other. +fn validate_network_options(options: &PacketSimulatorOptions) -> Result<(), String> { + if options.one_way_delay_mean < options.one_way_delay_min { + return Err(format!( + "one-way delay mean ({}) is below the minimum ({}); the exponential draw is \ + floored at the minimum, so the mean would never take effect", + options.one_way_delay_mean, options.one_way_delay_min, + )); + } + Ok(()) +} + fn main() { let args = Args::parse(); + // Server-side diagnostics (`emit_partition_diag` and friends) are the only + // record of a request the server dropped after logging, which is the shape that + // wedges a client's in-flight slot. Without a subscriber they go nowhere and the + // run looks like an unexplained stall, so install one and let `RUST_LOG` select. + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .init(); + // A provided seed reproduces a prior run exactly; otherwise draw one and // log it. Both the network and workload PRNGs derive from it. let seed = args.seed.unwrap_or_else(rand::random); let ticks = args.ticks; let clients = args.clients; let replicas = args.replicas; + let plane = args.plane; let crash_prob = args.crash_prob; let quiesce = !args.no_quiesce; // Surface the seed on any panic (invariant or oracle violation) so the run // is replayable. The process still exits non-zero via the default hook. std::panic::set_hook(Box::new(move |info| { - eprintln!("workload-fuzz FAILED — reproduce with --seed {seed}\n{info}"); + eprintln!("workload-fuzz FAILED, reproduce with --seed {seed}\n{info}"); + // The hook REPLACES the default one, so without this `RUST_BACKTRACE=1` + // silently does nothing and an assertion deep in consensus reports only its + // message. Gated on the env var: always printing would bury a campaign. + if std::env::var_os("RUST_BACKTRACE").is_some_and(|value| value != "0") { + eprintln!("{}", std::backtrace::Backtrace::force_capture()); + } })); - println!( - "workload-fuzz: seed={seed} ticks={ticks} clients={clients} replicas={replicas} \ - crash_prob={crash_prob} quiesce={quiesce}" - ); + let network_opts = network_options(&args, replicas, clients, seed); + if let Err(error) = validate_network_options(&network_opts) { + eprintln!("workload-fuzz: invalid network configuration: {error}"); + std::process::exit(2); + } + print_run_banner(&args, seed, &network_opts); // poll_messages / reply paths panic without an initialized pool; disabled // pooling falls through to the system allocator. @@ -108,64 +543,282 @@ fn main() { bucket_capacity: 1, }); - let client_ids: Vec = (1..=u128::from(clients)).collect(); - let network_opts = PacketSimulatorOptions { - node_count: replicas, - client_count: clients, - seed, - ..PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new( - usize::from(replicas), - client_ids.iter().copied(), - network_opts, - ); - let sim_clients: Vec = client_ids.iter().map(|&id| SimClient::new(id)).collect(); - - let ns = IggyNamespace::new(1, 1, 0); - sim.init_partition(ns); - for client in &sim_clients { - sim.register_client_with_primary(client); - } + let (mut sim, sim_clients, ns) = build_cluster(&args, seed, replicas, clients, network_opts); let mut options = WorkloadOptions::new(seed, replicas, vec![ns]); options.client_count = clients; options.crash_per_tick_ratio = crash_prob; - options.weights = ActionWeights::new(&[(Action::SendMessages, 100)]); + options.restart_per_tick_ratio = args.restart_prob; + options.spare_primary = !args.crash_primary; + if let Some(min_survivors) = args.min_survivors { + options.min_survivors = min_survivors; + } + options.ack_quorum_ratio = args.ack_quorum_ratio; + options.weights = plane.weights(); let mut workload = Workload::new(options); - let replies = run(&mut sim, &mut workload, &sim_clients, ticks, u64::MAX); + let mut injector = FaultInjector::new(seed, replicas); + let replies = run_with_faults( + &mut sim, + &mut workload, + &sim_clients, + ticks, + u64::MAX, + &mut injector, + ); println!( - "ran {ticks} ticks; {replies} replies; crashed replicas: {}", - sim.crashed.len() + "ran {ticks} ticks; {replies} replies; crashes={} restarts={} still down: {}", + injector.crashes(), + injector.restarts(), + sim.crashed.len(), ); + // Printed before the quiesce assert, so a failed drain still reports what + // the run managed to do. Reading it after the assert meant the failure that + // most needs the numbers is the one that never shows them. + print_coverage(&workload); + if quiesce { - if oracle::drive_to_quiesce(&mut sim, &mut workload, 50_000) { - oracle::assert_converged(&sim, &workload); - println!("quiesced and converged (leader-relative + entity oracle)"); + // Liveness phase, before anything is asserted. A drain against a handicapped + // cluster has no verdict: a replica still down cannot answer or be compared + // against, and on a solo run leaves nobody to drain at all. TigerBeetle's VOPR + // does the same (`transition_to_liveness_mode`). + let revived: Vec = (0..replicas).filter(|idx| sim.is_crashed(*idx)).collect(); + for replica_idx in &revived { + sim.replica_restart(*replica_idx); + } + sim.network.heal(); + println!( + "liveness phase: network healed, {} replica(s) restarted {revived:?}", + revived.len(), + ); + + // A failed drain is a hard failure, not a warning. It was a warning while a + // lost request could not be retried, making stalls expected and + // unactionable; with the client resending, a request unanswered inside the + // budget is either a wedge or a liveness bug. + assert!( + oracle::drive_to_quiesce(&mut sim, &mut workload, 50_000), + "{}", + oracle::quiesce_failure_report(&sim, &workload), + ); + // Then wait for one agreed view before asserting. `assert_converged` resolves + // the leader as whichever live replica claims to be primary, so asserting + // mid-view-change finds none or finds a deposed one, both false failures. + assert!( + oracle::settle_to_stable_view(&mut sim, &mut workload, 50_000), + "metadata views never converged after the drain\n{}", + oracle::quiesce_failure_report(&sim, &workload), + ); + let convergence = oracle::assert_converged(&sim, &mut workload); + // Named, not implied. `assert_converged` skips the entity oracle when an + // eviction disarmed it and the shadow has not been proven consistent since, + // so "converged" alone would report a run that asserted nothing about entity + // state exactly like one that asserted everything. + let entity_oracle = if !workload.serial_run() { + "skipped (concurrent run)" + } else if workload.strict_outcome_oracle() { + "held" } else { + "DISARMED by an eviction and never re-armed" + }; + println!( + "quiesced and converged (leader-relative; entity oracle: {entity_oracle}; \ + evictions={}; ops_compared={} replicas_compared={} namespaces_checked={})", + workload.evictions(), + convergence.ops_compared, + convergence.replicas_compared, + convergence.namespaces_checked, + ); + assert!( + !args.require_entity_oracle || workload.strict_outcome_oracle(), + "--require-entity-oracle: the entity oracle was {entity_oracle}, so this run \ + proved nothing about entity state (seed={seed:#x})" + ); + let live = usize::from(replicas) - sim.crashed.len(); + assert!( + args.min_ops_compared == 0 + || live < 2 + || convergence.ops_compared >= args.min_ops_compared, + "--min-ops-compared {}: {live} replicas live but only {} op(s) witnessed \ + by more than one, so cross-replica agreement went untested \ + (seed={seed:#x})", + args.min_ops_compared, + convergence.ops_compared, + ); + // Again after the drain: the drain both answers outstanding requests and + // issues its own resends, so the pre-drain numbers are not the final ones. + print_coverage(&workload); + } + + // After the quiesce block, so the drain's own commits count. Rejections are added + // to the per-action counter, which tracks committed SUCCESSES: a business + // rejection is still an op the cluster ordered and applied as a no-op, so a run + // full of them exercised the plane. + let stats = workload.auditor.stats(); + let commits: u64 = stats.commits_per_action.iter().sum::() + stats.committed_rejections; + assert!( + commits >= args.min_commits, + "--min-commits {}: the run committed {commits} operation(s) on the {plane:?} \ + plane, so every oracle above compared empty against empty (seed={seed:#x})", + args.min_commits, + ); + if args.require_faults { + assert!( + crash_prob <= 0.0 || injector.crashes() > 0, + "--require-faults: --crash-prob {crash_prob} crashed nothing \ + (seed={seed:#x})" + ); + assert!( + args.restart_prob <= 0.0 || injector.restarts() > 0, + "--require-faults: --restart-prob {} restarted nothing (seed={seed:#x})", + args.restart_prob, + ); + } + + print_command_coverage(&sim); + println!("workload-fuzz: OK (seed={seed})"); +} + +/// Stand up the cluster, seed its namespace, and get every client a session. +/// +/// The shell path differs in two ways that have to agree: a partition request's +/// namespace resolves against committed metadata, so the stream and topic behind it +/// must exist and not just the partition group; and dispatch admits a request only +/// from a bound session, which only a login mints. +fn build_cluster( + args: &Args, + args_seed: u64, + replicas: u8, + clients: u8, + network_opts: PacketSimulatorOptions, +) -> (Simulator, Vec, IggyNamespace) { + let client_ids: Vec = (1..=u128::from(clients)).collect(); + // A bounded journal is what makes a checkpoint happen, and a checkpoint needs a + // data directory for the coordinator to write into. Neither exists on the plain + // constructors, which is why an unbounded run never reaches WAL drain, + // `RangeEvicted` or metadata state transfer. The directory is leaked + // deliberately: it holds the snapshots a failing run is diagnosed from. + let mut sim = match args.journal_slots { + Some(slots) => { + let root = checkpoint_data_dir(args, args_seed); println!( - "WARN: did not quiesce within budget — expected when crashing to bare quorum \ - or under the metadata request-gap limitation; per-tick invariants still held" + "checkpointing enabled: journal_slots={slots} data_dir={}", + root.display() ); + let sim = Simulator::with_checkpoints( + usize::from(replicas), + client_ids.iter().copied(), + network_opts, + args.shell, + &root, + ); + sim.set_metadata_journal_slots(slots); + sim } + None if args.shell => Simulator::with_shards_shell( + usize::from(replicas), + 1, + client_ids.iter().copied(), + network_opts, + ), + None => Simulator::new( + usize::from(replicas), + client_ids.iter().copied(), + network_opts, + ), + }; + sim.set_restore_partition_frontier(args.restore_partition_frontier); + let sim_clients: Vec = client_ids.iter().map(|&id| SimClient::new(id)).collect(); + + let ns = IggyNamespace::new(1, 1, 0); + sim.init_partition(ns); + if args.shell { + sim.seed_stream_topic_partition(ns); } + for client in &sim_clients { + if args.shell { + sim.shell_login(client); + } else { + sim.register_client_with_primary(client); + } + } + (sim, sim_clients, ns) +} +/// Where a checkpointing run keeps its snapshots. +/// +/// Per-process, not per-seed: a restart now recovers from `snapshot.bin`, so a +/// directory an earlier run left is INPUT to this one and seeded naming would let a +/// run silently boot off its predecessor's state while reporting only the seed. +fn checkpoint_data_dir(args: &Args, seed: u64) -> std::path::PathBuf { + let root = match &args.data_dir { + Some(explicit) => { + assert!( + args.reuse_data_dir || !explicit.exists(), + "--data-dir {} already exists; the run would boot off what it holds. \ + Remove it, name another, or pass --reuse-data-dir.", + explicit.display(), + ); + explicit.clone() + } + None => std::env::temp_dir().join(format!( + "iggy-workload-fuzz-{seed:#x}-{}", + std::process::id() + )), + }; + std::fs::create_dir_all(&root).expect("fuzz data directory must be creatable"); + root +} + +/// Which protocol commands the run delivered, and which it never reached. +/// +/// The harness wires far more of the command space than any one scenario drives, and +/// "is this path covered?" was previously answered by grepping the source. Counted +/// at delivery, so a command listed here really arrived somewhere. +fn print_command_coverage(sim: &Simulator) { + let counts = sim.network.command_counts(); + let mut seen: Vec = Vec::new(); + let mut unseen: Vec<&str> = Vec::new(); + for (discriminant, &count) in counts.iter().enumerate() { + let label = COMMAND_LABELS[discriminant]; + if label == "Reserved" { + continue; + } + if count > 0 { + seen.push(format!("{label}={count}")); + } else { + unseen.push(label); + } + } + println!("commands delivered: {}", seen.join(" ")); + println!("commands never delivered: {}", unseen.join(" ")); +} + +/// Reply, rejection and resend counters plus per-action commits. +fn print_coverage(workload: &Workload) { let stats = workload.auditor.stats(); println!( - "coverage: replies_seen={} replies_unknown={} committed_rejections={} samples_none={}", + "coverage: replies_seen={} replies_unknown={} committed_rejections={} \ + samples_none={} resends={} denials={} transients={} evictions={}", stats.replies_seen, stats.replies_unknown, stats.committed_rejections, workload.samples_none(), + workload.resends(), + stats.denials, + stats.transient_rejections, + workload.evictions(), ); for action in Action::iter() { let commits = stats.commits(action); - if commits > 0 { - println!(" {action:?}: {commits} commits"); + let (refused, code) = stats.denials_per_action[action as usize]; + let (transients, transient_code) = stats.transient_rejections_per_action[action as usize]; + if commits > 0 || refused > 0 || transients > 0 { + println!( + " {action:?}: {commits} commits, {refused} denied (last status {code}), \ + {transients} transient (last code {transient_code})" + ); } } - - println!("workload-fuzz: OK (seed={seed})"); } diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index 3e7742ed1e..6f19bd1077 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs @@ -30,6 +30,10 @@ use iggy_binary_protocol::requests::messages::{ use iggy_binary_protocol::requests::partitions::{ CreatePartitionsRequest, DeletePartitionsRequest, }; +use iggy_binary_protocol::requests::personal_access_tokens::{ + CreatePersonalAccessTokenRequest as WireCreatePersonalAccessTokenRequest, + DeletePersonalAccessTokenRequest as WireDeletePersonalAccessTokenRequest, +}; use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; use iggy_binary_protocol::requests::streams::{ CreateStreamRequest, DeleteStreamRequest, PurgeStreamRequest, UpdateStreamRequest, @@ -77,6 +81,17 @@ pub struct SimClient { /// a pure function of the seed. See [`SimClient::next_message_id`]. message_counter: Cell, session: Cell, + /// Whether this client talks to the server's real dispatch layer, which + /// changes what a PAT request must contain. + /// + /// A real client sends `[name][expiry]` and the server mints the token and its + /// hash in `maybe_rewrite_pat_request`, rewriting the request into the + /// replicated form before consensus sees it. The raw path has no dispatch layer + /// and so no rewrite, so a request submitted there must arrive already + /// replicated. Sessions split the same way (`register` raw, `login` shell); + /// this is the one op family whose BODY differs rather than its envelope. Set + /// by `Simulator::shell_login_via`, so it follows the path the client took. + shell_wire: Cell, } impl SimClient { @@ -88,9 +103,33 @@ impl SimClient { partition_counter: Cell::new(0), message_counter: Cell::new(0), session: Cell::new(0), + shell_wire: Cell::new(false), } } + /// Mark this client as talking to the real dispatch layer, so PAT requests + /// carry the client wire shape rather than the replicated one. See + /// [`SimClient::shell_wire`]. + pub fn set_shell_wire(&self) { + self.shell_wire.set(true); + } + + /// Put this client back on the replicated wire shape. + /// + /// The inverse exists because the flip is otherwise permanent and silent: a + /// client moved to the client wire shape by mistake stops covering the + /// replicated PAT path for the rest of the run, with nothing failing. + pub fn clear_shell_wire(&self) { + self.shell_wire.set(false); + } + + /// Whether this client talks the client wire shape (see + /// [`SimClient::set_shell_wire`]). + #[must_use] + pub const fn shell_wire(&self) -> bool { + self.shell_wire.get() + } + #[must_use] pub const fn client_id(&self) -> u128 { self.client_id @@ -120,10 +159,11 @@ impl SimClient { /// Assign the wire request id for `operation`, keyed by plane. /// - /// Metadata/replicated ops advance a contiguous `1, 2, 3, …` counter: the - /// `ClientTable` dedups them and rejects anything but `committed + 1`, so a - /// gap opens a permanent `RequestGap` and wedges the client's metadata - /// plane. Partition ops are at-least-once with no dedup and the server + /// Metadata/replicated ops advance a contiguous `1, 2, 3, …` counter, matching + /// the real SDK. Gaps are admitted rather than fatal (`check_request` answers + /// `New` to anything above the watermark; there is no `RequestGap`), but the + /// dedup ring is sized for a contiguous sequence. Partition ops are at-least-once + /// with no dedup and the server /// treats their id as an opaque echo, so they draw from a separate counter /// offset into a disjoint range ([`PARTITION_ID_BASE`]). A partition id can /// therefore never equal a metadata id, so a delayed or duplicated partition @@ -229,6 +269,18 @@ impl SimClient { .expect("login request must be valid") } + /// Tear down this client's bound session. + /// + /// Replicates through the metadata plane like any other session op, so it + /// carries the bound session and a metadata request id and needs no body. A + /// logout to a BACKUP is what produces `ForwardLogout`: the backup owns the + /// connection but not the log, so it asks the primary to commit the teardown + /// and answers once `ForwardLogoutResult` returns. + #[must_use] + pub fn logout(&self) -> Message { + self.build_request(Operation::Logout, &[]) + } + /// # Panics /// Panics if the stream name is not a valid wire name. pub fn create_stream(&self, name: &str) -> Message { @@ -487,14 +539,22 @@ impl SimClient { name: &str, expiry: u64, ) -> Message { + let name = WireName::new(name).expect("PAT name must be valid"); + // Through dispatch, send what a real client sends: the server resolves the + // acting user from the session and mints the token and its hash in + // `maybe_rewrite_pat_request`, rewriting this into the replicated form + // before consensus sees it. A client cannot produce that form, not knowing + // the hash, so sending it here made every PAT request fail to decode. + if self.shell_wire.get() { + let wire = WireCreatePersonalAccessTokenRequest { name, expiry }; + return self.build_request(Operation::CreatePersonalAccessToken, &wire.to_bytes()); + } + // Raw path: no dispatch layer, so no rewrite ever happens and the request + // has to arrive already replicated. let wire = CreatePersonalAccessTokenRequest { user_id: 0, - name: WireName::new(name).expect("PAT name must be valid"), + name, expiry, - // Deterministic stub for the simulator. Production servers mint - // this in `maybe_rewrite_pat_request` on the primary; the - // simulator drives the wire path directly without that rewrite - // step. token_hash: [b'a'; 64], }; self.build_request(Operation::CreatePersonalAccessToken, &wire.to_bytes()) @@ -503,9 +563,15 @@ impl SimClient { /// # Panics /// Panics if `name` is not a valid `WireName`. pub fn delete_personal_access_token(&self, name: &str) -> Message { + let name = WireName::new(name).expect("PAT name must be valid"); + // See `create_personal_access_token` for why the shape depends on the path. + if self.shell_wire.get() { + let wire = WireDeletePersonalAccessTokenRequest { name }; + return self.build_request(Operation::DeletePersonalAccessToken, &wire.to_bytes()); + } let wire = DeletePersonalAccessTokenRequest { user_id: 0, - name: WireName::new(name).expect("PAT name must be valid"), + name, only_if_expired: false, }; self.build_request(Operation::DeletePersonalAccessToken, &wire.to_bytes()) diff --git a/core/simulator/src/deps.rs b/core/simulator/src/deps.rs index 7c73f1a41a..39b12c8454 100644 --- a/core/simulator/src/deps.rs +++ b/core/simulator/src/deps.rs @@ -27,6 +27,7 @@ use metadata::stm::user::Users; use server_common::{Message, iobuf::Owned}; use std::cell::{Cell, RefCell, UnsafeCell}; use std::collections::HashMap; +use std::ops::RangeInclusive; /// Fixed synthetic epoch for [`SimClock`]: 2026-01-01T00:00:00Z in micros. /// @@ -104,6 +105,14 @@ pub struct SimJournal { /// retained head in O(1) without scanning `headers` (see /// [`SimJournal::last_op`]). last_op: Cell>, + /// Snapshot watermark. A real value here is what makes `RangeEvicted` + /// reachable; see the `Journal::snapshot_op` impl. + snapshot_op: Cell, + /// Slots this journal pretends to have, or `None` for unbounded. + /// + /// `SnapshotCoordinator::should_checkpoint` gates on `remaining_capacity`, so + /// unbounded means no checkpoint ever. A test wanting one sets a small count. + slot_count: Cell>, /// Debug-only single-accessor tripwire. `entry` / `append` hold a /// [`JournalAccessGuard`] across their whole body, including the storage /// `.await`, so if a suspending storage tier ever let a second task touch @@ -121,6 +130,8 @@ impl Default for SimJournal { offsets: UnsafeCell::new(HashMap::new()), write_offset: Cell::new(0), last_op: Cell::new(None), + snapshot_op: Cell::new(0), + slot_count: Cell::new(None), #[cfg(debug_assertions)] accessing: Cell::new(false), } @@ -180,6 +191,23 @@ impl>> Journal for SimJournal { self.last_op.get() } + /// Slots left before a checkpoint is forced, mirroring + /// `PrepareJournal::remaining_capacity`: the ring holds `slot_count`, everything + /// at or below the watermark is reclaimable, so `last_op - snapshot_op` is + /// occupied. `None` while unbounded, which `should_checkpoint` reads as never. + fn remaining_capacity(&self) -> Option { + let slot_count = self.slot_count.get()?; + let Some(last) = self.last_op.get() else { + return Some(slot_count); + }; + let snapshot = self.snapshot_op.get(); + if last <= snapshot { + return Some(slot_count); + } + let used = usize::try_from(last - snapshot).unwrap_or(usize::MAX); + Some(slot_count.saturating_sub(used)) + } + /// Drop the suffix, so a simulated backup whose entries disagree with a started /// view reconciles the way a real one does. Mirrors /// `PrepareJournal::truncate_from`, whose watermark stays put; here it never moves. @@ -207,15 +235,31 @@ impl>> Journal for SimJournal { Ok(doomed.len()) } - /// The simulated journal retains everything for the run, so nothing is - /// ever superseded by a snapshot. Answered explicitly (the trait has no - /// default) so a simulated state transfer has to opt into a watermark - /// rather than silently inherit one that never moves. + /// The snapshot watermark: entries at or below it are evictable. + /// + /// Load-bearing even though this journal retains every entry. The repair server + /// floors what it serves at `snapshot_op + 1` and announces the skipped prefix + /// as `RangeEvicted`, the only route into state transfer, so a constant 0 left + /// every state-transfer frame unreachable however the harness was driven. fn snapshot_op(&self) -> u64 { - 0 + self.snapshot_op.get() } - fn set_snapshot_op(&self, _op: u64) {} + /// Advance the watermark. Production only moves it forward, on a checkpoint or a + /// transfer install; a retreating floor would re-offer ops the serving side has + /// told a peer are gone. + /// + /// # Panics + /// If `op` is below the current watermark, as `PrepareJournal` asserts. Maxing + /// silently would leave the simulator the one place a retreat survives. + fn set_snapshot_op(&self, op: u64) { + let current = self.snapshot_op.get(); + assert!( + op >= current, + "snapshot_op must be monotonically increasing: {current} -> {op}" + ); + self.snapshot_op.set(op); + } // TODO(hubcio): validate that the caller's checksum matches the stored // header - currently this looks up by op only, ignoring the checksum. @@ -280,6 +324,74 @@ impl>> Journal for SimJournal { let headers = unsafe { &*self.headers.get() }; headers.get(&(idx as u64)) } + + /// Reclaim the prefix a checkpoint superseded, advancing the watermark to the + /// end of the drained range. + /// + /// Required, not inherited: the trait's default drains nothing, so a simulated + /// checkpoint left the whole WAL in place, a peer's repair found every op it + /// asked for, and arming the coordinator alone still produced no `RangeEvicted`. + /// + /// The watermark moves last, as in `PrepareJournal::drain`: advancing it before + /// the entries are gone would make live entries look evictable. + async fn drain(&self, ops: RangeInclusive) -> std::io::Result> { + #[cfg(debug_assertions)] + let _guard = JournalAccessGuard::new(&self.accessing); + let end_op = *ops.end(); + let doomed: Vec = { + let headers = unsafe { &*self.headers.get() }; + let mut doomed: Vec = headers + .keys() + .copied() + .filter(|op| ops.contains(op)) + .collect(); + // Sorted: the trait promises op order, and hash order would make a + // replay of this drain diverge. + doomed.sort_unstable(); + doomed + }; + + let mut drained = Vec::with_capacity(doomed.len()); + for op in doomed { + // Read before removing, through `Storage` rather than the + // `MemStorage`-only sync path, so this stays generic. The borrow does + // NOT span the read, unlike `entry`'s: the block ends it and yields only + // `Copy` data, so the `.await` holds no reference into the `UnsafeCell`. + // Deliberate, since `drain` invalidates every outstanding + // `header`/`previous_header` reference too. + let located = { + let headers = unsafe { &*self.headers.get() }; + let offsets = unsafe { &*self.offsets.get() }; + headers + .get(&op) + .and_then(|header| offsets.get(&op).map(|offset| (header.size, *offset))) + }; + // Propagated, not swallowed. Dropping the entry loses a WAL record while + // reporting a successful drain; `PrepareJournal` returns the error and + // poisons itself, and a harness surviving what production refuses to + // cannot find the bug this path exists to catch. + if let Some((size, offset)) = located { + let buffer = self.storage.read_at(offset, vec![0; size as usize]).await?; + let message = + Message::try_from(Owned::<4096>::copy_from_slice(&buffer)).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("drain: op {op} does not decode as a prepare"), + ) + })?; + drained.push(message); + } + let headers = unsafe { &mut *self.headers.get() }; + let offsets = unsafe { &mut *self.offsets.get() }; + headers.remove(&op); + offsets.remove(&op); + } + + if end_op > self.snapshot_op.get() { + self.snapshot_op.set(end_op); + } + Ok(drained) + } } impl JournalHandle for SimJournal { @@ -298,6 +410,13 @@ impl SimJournal { self.last_op.get() } + /// Bound this journal to `slots`, so running low forces a checkpoint. Unbounded + /// by default (see `slot_count`), and then nothing produces the snapshot a state + /// transfer serves. + pub fn set_slot_count(&self, slots: usize) { + self.slot_count.set(Some(slots)); + } + /// Forget one op, leaving a hole exactly where a lost prepare would. /// /// Tests only. The alternative is choreographing `Prepare`, `Commit` and @@ -322,17 +441,29 @@ impl SimJournal { /// `commit` any journaled prepare stamped, a lower bound, since a prepare /// records the primary's commit point at send time, so the true point may be one /// op higher and re-commits on rejoin. + /// + /// Floored at the snapshot watermark and clamped at the first gap above it, as + /// `metadata::recover` folds from `snapshot_floor` and stops at `chain_break_op`. + /// The fold only sees surviving headers, so a backup missing one prepare would + /// otherwise claim a commit point ABOVE the hole, telling the cluster there is + /// nothing to repair. #[must_use] pub fn recovery_commit_watermark(&self, solo: bool) -> u64 { - if solo { - return self.last_op.get().unwrap_or(0); - } + let floor = self.snapshot_op.get(); let headers = unsafe { &*self.headers.get() }; - headers - .values() - .map(|header| header.commit) - .max() - .unwrap_or(0) + let claimed = if solo { + self.last_op.get().unwrap_or(0).max(floor) + } else { + headers + .values() + .map(|header| header.commit) + .fold(floor, u64::max) + }; + let mut watermark = floor; + while watermark < claimed && headers.contains_key(&(watermark + 1)) { + watermark += 1; + } + watermark } /// The head prepare's header, `None` when empty. Restores the last-prepare diff --git a/core/simulator/src/executor/mod.rs b/core/simulator/src/executor/mod.rs index 8fdfc89a50..5a4e2c2f75 100644 --- a/core/simulator/src/executor/mod.rs +++ b/core/simulator/src/executor/mod.rs @@ -34,7 +34,7 @@ pub mod time; use futures::task::ArcWake; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use rand_xoshiro::rand_core::SeedableRng; use std::cell::RefCell; use std::collections::BTreeMap; @@ -47,12 +47,6 @@ use std::time::Duration; pub use time::{SimInstant, TimerHandle}; -/// Salt for deriving the executor's PRNG stream from the simulation seed. -/// -/// Sibling of the workload fault salt (`0x5A1A_F0E5_FACE_0001`): independent -/// streams keep scheduling draws from perturbing network or workload traces. -pub const EXECUTOR_SEED_SALT: u64 = 0x5A1A_F0E5_FACE_0002; - /// Ready on the second poll; the first re-queues the task behind every other /// ready task, exactly like a wake arriving mid-await. Lets a task give the /// executor a turn without parking on a timer, which would only wake it at @@ -107,7 +101,7 @@ pub struct DetExecutor { /// Ready queue in insertion order; the next victim is a seeded uniform /// pick, so the schedule is a pure function of (seed, wake history). ready: Vec, - rng: Xoshiro256Plus, + rng: Xoshiro256PlusPlus, wake: Arc, timer: TimerHandle, /// Futures staged by task code through `MessageBus::spawn` (the sim's @@ -130,7 +124,7 @@ impl DetExecutor { tasks: Vec::new(), free: Vec::new(), ready: Vec::new(), - rng: Xoshiro256Plus::seed_from_u64(seed ^ EXECUTOR_SEED_SALT), + rng: Xoshiro256PlusPlus::seed_from_u64(crate::seeds::SimSeeds::derive(seed).executor), wake: Arc::new(WakeQueue { woken: Mutex::new(Vec::new()), }), diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 8e971c7083..426aa483a0 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -23,6 +23,7 @@ pub mod network; pub mod packet; pub mod ready_queue; pub mod replica; +pub mod seeds; pub mod workload; use bus::SimOutbox; @@ -32,17 +33,21 @@ use deps::SimClock; use deps::SimSuperblock; use deps::{MemStorage, SimJournal}; use executor::{DetExecutor, RunOutcome, TaskId}; -use iggy_binary_protocol::{GenericHeader, ReplyHeader}; +use iggy_binary_protocol::{Command, GenericHeader, ReplyHeader}; use iggy_common::IggyError; use message_bus::installer::conn_info::{ClientConnMeta, ClientTransportKind}; use metadata::impls::metadata::StreamsFrontend; use network::Network; use packet::{PacketSimulatorOptions, ProcessId}; -use partitions::{Partition, PartitionOffsets, PollFragments, PollingArgs, PollingConsumer}; +use partitions::{ + Partition, PartitionOffsets, PollFragments, PollingArgs, PollingConsumer, + RetainedPartitionState, +}; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use rand_xoshiro::rand_core::SeedableRng; use replica::{Replica, SIM_INBOX_CAPACITY, new_shard}; +use seeds::SimSeeds; use server_common::Message; use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; use shard::CONSENSUS_TICK_INTERVAL; @@ -52,51 +57,51 @@ use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::rc::Rc; -/// Poll budget per [`DetExecutor::run_until_stalled`] call. The pumps are -/// event-driven, so hitting this means a task is spin-waking: always a bug, -/// surfaced as a panic carrying the seed. +/// Poll budget per [`DetExecutor::run_until_stalled`]. Pumps are event-driven, so +/// hitting it means a task is spin-waking: a bug, panicked with the seed. const POLL_BUDGET: u32 = 100_000; -/// Salt for the entry-shard PRNG stream (sibling of the workload fault salt -/// and [`executor::EXECUTOR_SEED_SALT`]). +/// Retry interval and total budget for a setup handshake +/// (`register_client_with_primary`, `shell_login`). /// -/// In production the shard-0 coordinator round-robins inbound connections -/// across shards, so the shard that receives a peer's bytes is unrelated to -/// the shard owning the target consensus group. The sim models that homing -/// with a seeded uniform pick per delivered packet; an independent stream -/// keeps those draws from perturbing network or workload traces. -pub const ENTRY_SHARD_SEED_SALT: u64 = 0x5A1A_F0E5_FACE_0003; - -/// One simulated replica: its shards plus the executor bookkeeping needed -/// to crash it. One entry per shard in `shards`/`pump_tasks` (a single -/// shard until multi-shard lands). +/// Submit-once dies under injected loss. Both are metadata ops with a stable request +/// id, so the client table dedups the resend. +const SETUP_RETRY_STEPS: u32 = 50; +const SETUP_TOTAL_STEPS: u32 = 4_000; + +/// One simulated replica: shards plus the executor bookkeeping to crash it. One +/// entry per shard in `shards` / `pump_tasks`. pub struct SimReplica { /// Shards of this replica, indexed by shard id. pub shards: Vec>, - /// Shard 0's durable superblock, held here rather than inside the shard so its - /// bytes survive the shard being dropped and rebuilt across a restart. + /// Shard 0's durable superblock. Held here, not in the shard, so its bytes + /// survive the shard being rebuilt across a restart. pub superblock: Rc, - /// Shard 0's metadata WAL, held here for the same reason as the superblock: the - /// bytes and index survive a restart, so a rebuilt replica recovers its - /// op/commit and committed metadata from its own disk. Only shard 0 owns - /// metadata consensus, so this is the single retained journal. + /// Shard 0's metadata WAL, retained across a restart like `superblock`, so a + /// rebuilt replica recovers op/commit and committed metadata from its own disk. + /// Shard 0 owns the only metadata consensus, so this is the only journal. pub metadata_journal: Rc>, - /// Shard 0's metadata consensus incarnation nonce. Harness-owned: a seed-derived - /// value bumped by one on each restart, so successive incarnations are distinct - /// yet the run stays byte-identical on replay. See + /// Shard 0's metadata incarnation nonce. Seed-derived, bumped by one per + /// restart: distinct incarnations, byte-identical replay. See /// `VsrConsensus::set_incarnation`. pub metadata_incarnation: u128, - /// One durable superblock per partition group this replica has materialised, - /// harness-owned for the same reason as the metadata one: the bytes survive - /// the shards being dropped and rebuilt, so a re-materialised group recovers - /// the `(view, log_view)` it recorded instead of re-entering view 0. Without - /// a store the persist gate marks every view durable without writing, which - /// leaves the gate, its write-failure fence, and view recovery all - /// unexercised. + /// One durable superblock per materialised partition group, retained like + /// `superblock` so a re-materialised group recovers its recorded + /// `(view, log_view)` instead of re-entering view 0. Storeless, the persist gate + /// marks every view durable without writing, leaving the gate, its write-failure + /// fence and view recovery unexercised. pub partition_superblocks: RefCell>>, - /// Keeps each pump's stop channel alive; dropping one would end that - /// pump gracefully, which is reserved for future shutdown/restart - /// tests (crash uses `DetExecutor::abort` instead). + /// One retained message log per partition group, plus the offsets recovered from + /// it. A real server's messages are in segment files; the simulator has none, so + /// without this a rebuilt partition comes back empty and the monotonicity + /// invariant calls a discarded log a consensus regression. Populated by + /// [`Simulator::replica_restart`], consumed by `materialise_partition`. + partition_logs: RefCell>, + /// This replica's data directory when checkpoints are enabled. Retained so a + /// restart reads back the snapshot its previous incarnation wrote. + data_dir: Option, + /// Keeps each pump's stop channel alive. Dropping one ends that pump gracefully, + /// reserved for shutdown tests; crash uses `DetExecutor::abort`. _stop_txs: Vec>, /// Pump task per shard, aborted on crash. pump_tasks: Vec, @@ -107,8 +112,7 @@ impl SimReplica { /// deterministic hash the router uses. /// /// # Panics - /// Panics if the shard count does not fit `u32` (impossible: mesh - /// construction caps it at `u16`). + /// If the shard count does not fit `u32`; mesh construction caps it at `u16`. #[must_use] pub fn partition_shard(&self, namespace: IggyNamespace) -> &Rc { let shard_count = u32::try_from(self.shards.len()).expect("shard count fits u32"); @@ -117,33 +121,63 @@ impl SimReplica { } } +/// One replica's view of a partition group's consensus. Read by the quiesce oracle; +/// see [`Simulator::partition_consensus_state`]. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PartitionConsensusState { + pub status: consensus::Status, + pub view: u32, + pub is_primary: bool, + /// Ops committed in the group. Not `PartitionOffsets::commit_offset`, the + /// highest durably PERSISTED offset, which counts an uncommitted suffix. + pub commit_min: u64, +} + pub struct Simulator { - /// All replicas, indexed by replica id. Always fully populated — crashed - /// replicas are kept alive but skipped during dispatch. + /// All replicas, indexed by replica id. Always fully populated; crashed replicas + /// stay alive but are skipped during dispatch. pub replicas: Vec, - /// Per-replica outbox, indexed by replica id. Shared with consensus inside - /// each replica via [`SharedSimOutbox`](bus::SharedSimOutbox). + /// Per-replica outbox, indexed by replica id. Shared with consensus via + /// [`SharedSimOutbox`](bus::SharedSimOutbox). pub outboxes: Vec>, - /// Set of replica ids that are currently crashed. Dispatch and outbox drain - /// are skipped for these ids. + /// Currently-crashed replica ids. Dispatch and outbox drain skip these. pub crashed: HashSet, pub network: Network, pub replica_count: u8, pub client_ids: Vec, - /// Drives every shard pump; scheduling picks and virtual time both - /// derive from the network seed, so the schedule replays with it. + /// Drives every shard pump. Scheduling picks and virtual time both derive from + /// the seed, so the schedule replays with it. executor: DetExecutor, - /// Picks which shard of a replica receives each inbound packet, - /// modeling the coordinator's connection homing (see - /// [`ENTRY_SHARD_SEED_SALT`]). - entry_rng: Xoshiro256Plus, + /// Picks which shard receives each inbound packet, modelling the coordinator's + /// connection homing: production round-robins inbound connections, so the shard + /// receiving a peer's bytes is unrelated to the one owning the target group. Own + /// stream ([`SimSeeds::entry_shard`]) so one draw per delivered packet cannot + /// perturb the network or workload traces. + entry_rng: Xoshiro256PlusPlus, /// Network seed, kept for livelock diagnostics. seed: u64, - /// Dispatch-shell mode: when set, inbound client packets are delivered - /// through the real `on_client_request` handler (see - /// [`shard::IggyShard::deliver_client_request`]) instead of the raw - /// `dispatch` routing. Chosen at construction via - /// [`Simulator::with_shards_shell`]. + /// Clients the cluster has evicted since the last drain, in delivery order. + /// + /// An eviction ends a session: outstanding requests go unanswered and the client + /// must log in again. Recorded, not acted on, because re-establishing a session + /// steps the simulator and drops workload expectations, neither of which belongs + /// inside packet delivery. + evicted: Vec, + /// Whether a rebuilt partition recovers its consensus frontier from the carried + /// log. OFF by default: production restores the view alone (`load_partition`), so + /// a run with it on studies a system more durable than Iggy is. See + /// `IggyShard::init_partition`. + restore_partition_frontier: bool, + /// Replies a setup handshake pulled off the wire that were not its own. + /// + /// `await_setup_reply` steps the whole simulator, so it sees every client's + /// replies. Dropping the rest strands their auditor expectations until a resend + /// re-commits, so they are parked here and returned by the next [`Self::step`]. + deferred_client_replies: Vec>, + /// Dispatch-shell mode: inbound client packets go through the real + /// `on_client_request` handler (see + /// [`shard::IggyShard::deliver_client_request`]) instead of raw `dispatch` + /// routing. Set at construction by [`Simulator::with_shards_shell`]. shell: bool, } @@ -151,10 +185,9 @@ impl Simulator { /// New simulator with per-replica outboxes routed through a [`Network`]. /// /// # Panics - /// Panics if `clients` yields duplicate `client_id`s. The auditor - /// keys in-flight entries by `(client_id, request)` and the network - /// indexes packet routes by `client_id`; duplicates would collide on - /// both. + /// If `clients` yields duplicate `client_id`s. The auditor keys in-flight + /// entries by `(client_id, request)` and the network indexes routes by + /// `client_id`; duplicates collide on both. pub fn new( replica_count: usize, clients: impl Iterator, @@ -163,12 +196,12 @@ impl Simulator { Self::with_shards(replica_count, 1, clients, network_options) } - /// [`Simulator::new`] with `shards_per_replica` shards on every replica, - /// meshed exactly like the server bootstrap: metadata plane on shard 0, - /// partitions hash-assigned, one pump task per shard. + /// [`Simulator::new`] with `shards_per_replica` shards per replica, meshed as + /// the server bootstrap does: metadata plane on shard 0, partitions + /// hash-assigned, one pump task per shard. /// /// # Panics - /// Panics on duplicate `client_id`s (see [`Simulator::new`]) or + /// On duplicate `client_id`s (see [`Simulator::new`]) or /// `shards_per_replica == 0`. pub fn with_shards( replica_count: usize, @@ -185,13 +218,13 @@ impl Simulator { ) } - /// [`Simulator::with_shards`] with the deterministic dispatch shell on: - /// every shard wires the server's real dispatch handlers, so a client - /// request runs as a task the seeded executor interleaves with the - /// pump. Off (the default) keeps the raw-`on_message` fast path. + /// [`Simulator::with_shards`] with the dispatch shell on: every shard wires the + /// server's real dispatch handlers, so a client request runs as a task the seeded + /// executor interleaves with the pump. Off, the default, keeps the raw + /// `on_message` fast path. /// /// # Panics - /// Panics on duplicate `client_id`s or `shards_per_replica == 0`. + /// On duplicate `client_id`s or `shards_per_replica == 0`. pub fn with_shards_shell( replica_count: usize, shards_per_replica: u16, @@ -207,6 +240,50 @@ impl Simulator { ) } + /// [`Simulator::new`] with checkpoints enabled, each replica rooted at + /// `/replica-N`. + /// + /// A data directory arms the metadata `SnapshotCoordinator`; without one + /// `checkpoint_if_needed` returns immediately and nothing produces the snapshot a + /// state transfer serves. + /// + /// Opt-in, and separate from the other constructors, because the coordinator + /// persists through `std::fs`: a harness that touches nothing outside memory + /// should not start writing files by omission. Writes are synchronous and never + /// touch the executor, so replay stays deterministic; the caller owns the + /// directory's lifetime. + /// + /// Pair with [`Simulator::set_metadata_journal_slots`]: a checkpoint is forced by + /// the journal running low on slots, and it is unbounded until told otherwise. + /// + /// # Panics + /// On duplicate `client_id`s, or if the per-replica directories cannot be + /// created. + pub fn with_checkpoints( + replica_count: usize, + clients: impl Iterator, + network_options: PacketSimulatorOptions, + shell: bool, + data_dir_root: &std::path::Path, + ) -> Self { + Self::build_inner( + replica_count, + 1, + clients, + network_options, + shell, + Some(data_dir_root), + ) + } + + /// Bound every replica's metadata journal to `slots`, so filling it forces a + /// checkpoint. See [`deps::SimJournal::set_slot_count`]. + pub fn set_metadata_journal_slots(&self, slots: usize) { + for replica in &self.replicas { + replica.metadata_journal.set_slot_count(slots); + } + } + #[allow(clippy::cast_possible_truncation, clippy::too_many_lines)] fn build( replica_count: usize, @@ -214,6 +291,25 @@ impl Simulator { clients: impl Iterator, network_options: PacketSimulatorOptions, shell: bool, + ) -> Self { + Self::build_inner( + replica_count, + shards_per_replica, + clients, + network_options, + shell, + None, + ) + } + + #[allow(clippy::cast_possible_truncation, clippy::too_many_lines)] + fn build_inner( + replica_count: usize, + shards_per_replica: u16, + clients: impl Iterator, + network_options: PacketSimulatorOptions, + shell: bool, + data_dir_root: Option<&std::path::Path>, ) -> Self { assert!( shards_per_replica >= 1, @@ -240,8 +336,8 @@ impl Simulator { let mut executor = DetExecutor::new(seed); let timer = executor.timer(); let spawns = executor.spawner(); - // One virtual clock for every consensus group: prepare timestamps - // become a pure function of the seed instead of the wall clock. + // One virtual clock for every consensus group, so prepare timestamps are a + // pure function of the seed rather than the wall clock. let consensus_clock = ConsensusClock::new(Rc::new(SimClock::new(timer.clone()))); let rc = replica_count as u8; @@ -258,18 +354,25 @@ impl Simulator { bus.add_replica(j); } let outbox = Rc::new(bus); - // Harness-owned so they outlive a replica restart, which drops and - // rebuilds the shards: the superblock's VSR state and the metadata WAL - // both survive. + // Harness-owned so the superblock's VSR state and the metadata WAL + // outlive a restart, which drops and rebuilds the shards. let superblock = Rc::new(SimSuperblock::default()); let metadata_journal = Rc::new(SimJournal::::default()); - // Initial incarnation, spread across the 128-bit space per replica so the - // restart increments in `replica_restart` never collide across replicas. - // Non-zero. + // Spread across the 128-bit space per replica so `replica_restart`'s + // increments never collide across replicas. Non-zero. let metadata_incarnation = 1 + (u128::from(id) << 64); + // Created up front: the snapshot coordinator writes into + // `/metadata/` and does not create it. Retained on `SimReplica` so a + // restart reads back the snapshot its previous incarnation wrote. + let replica_data_dir = data_dir_root.as_ref().map(|root| { + let dir = root.join(format!("replica-{i}")); + std::fs::create_dir_all(dir.join(metadata::impls::METADATA_DIR)) + .expect("simulator data directory is creatable"); + dir + }); - // One crossfire mesh per replica; every shard gets a clone of - // the canonical senders vec and exclusively takes its inbox. + // One crossfire mesh per replica. Every shard clones the canonical + // senders vec and exclusively takes its inbox. let (senders, mut inboxes, mut reply_inboxes) = shard::shard_mesh_channels( shards_per_replica, SIM_INBOX_CAPACITY, @@ -279,11 +382,10 @@ impl Simulator { let mut shards = Vec::with_capacity(usize::from(shards_per_replica)); let mut stop_txs = Vec::with_capacity(usize::from(shards_per_replica)); let mut pump_tasks = Vec::with_capacity(usize::from(shards_per_replica)); - // Single-writer metadata (mirrors the server bootstrap): shard 0 - // builds the writable STM and mints a factory bundle; every peer - // shard rebuilds a reader-mode mirror from it and sees committed - // metadata through the shared read handle. Shards are built in index - // order, so shard 0's bundle exists before any peer needs it. + // Single-writer metadata, as the server bootstrap does: shard 0 builds + // the writable STM and mints a factory bundle, every peer rebuilds a + // reader-mode mirror from it and reads committed metadata through the + // shared handle. Built in index order, so shard 0's bundle exists first. let mut metadata_bundle: Option = None; for shard_idx in 0..shards_per_replica { let inbox = inboxes[usize::from(shard_idx)] @@ -292,8 +394,8 @@ impl Simulator { let reply_inbox = reply_inboxes[usize::from(shard_idx)] .take() .expect("mesh yields exactly one reply inbox per shard"); - // Only shard 0 owns metadata consensus, so only it carries the - // superblock. Peer shards persist nothing. + // Shard 0 owns metadata consensus, so only it carries the + // superblock. Peers persist nothing. let shard_superblock = if shard_idx == 0 { Some(superblock.clone()) } else { @@ -316,6 +418,7 @@ impl Simulator { shard_journal, None, // fresh boot: no recovered VSR state metadata_incarnation, + (shard_idx == 0).then(|| replica_data_dir.clone()).flatten(), ); if shard_idx == 0 { metadata_bundle = Some( @@ -324,9 +427,8 @@ impl Simulator { ); } - // Same wiring as the server bootstrap: one pump task per - // shard, stopped only by the (held) stop channel or a - // crash abort. + // Server-bootstrap wiring: one pump task per shard, stopped only by + // the held stop channel or a crash abort. let (stop_tx, stop_rx) = shard::channel::<()>(1); let pump_shard = Rc::clone(&shard); pump_tasks.push(executor.spawn(async move { @@ -342,6 +444,8 @@ impl Simulator { metadata_journal, metadata_incarnation, partition_superblocks: RefCell::new(HashMap::new()), + partition_logs: RefCell::new(HashMap::new()), + data_dir: replica_data_dir, _stop_txs: stop_txs, pump_tasks, }); @@ -356,7 +460,10 @@ impl Simulator { replica_count: rc, client_ids, executor, - entry_rng: Xoshiro256Plus::seed_from_u64(seed ^ ENTRY_SHARD_SEED_SALT), + entry_rng: Xoshiro256PlusPlus::seed_from_u64(SimSeeds::derive(seed).entry_shard), + evicted: Vec::new(), + restore_partition_frontier: false, + deferred_client_replies: Vec::new(), seed, shell, } @@ -364,15 +471,14 @@ impl Simulator { /// Init a partition with its own consensus group on every live replica. /// - /// Mirrors the reconciler's outcome without running it: the namespace is - /// committed to metadata, the partition materialises only on its - /// hash-owning shard, and every shard of the replica gets the routing row - /// stamped with the committed `created_revision` (production seeds rows - /// through `ReconcileOp::{InsertOwned,InsertRouted}`). + /// The reconciler's outcome without running it: the namespace is committed to + /// metadata, the partition materialises only on its hash-owning shard, and every + /// shard gets the routing row stamped with the committed `created_revision`. + /// Production seeds rows through `ReconcileOp::{InsertOwned,InsertRouted}`. /// /// # Panics - /// Panics if a replica's shard count does not fit `u32` (impossible: - /// mesh construction caps it at `u16`). + /// If a replica's shard count does not fit `u32`; mesh construction caps it at + /// `u16`. // TODO(hubcio): partitions created down this path are built via // `IggyPartition::with_in_memory_storage` and rely on the writer-less // persist branch in `IggyPartition`; give them first-class in-memory @@ -383,19 +489,19 @@ impl Simulator { if self.crashed.contains(&(i as u8)) { continue; } - materialise_partition(replica, namespace); + materialise_partition(replica, namespace, self.restore_partition_frontier); } } - /// Seed the metadata `Streams` STM on each live replica's shard-0 writer - /// so a poll's namespace resolution (`resolve_partition_namespace`) - /// succeeds for `namespace`, which the partition-plane-only - /// [`Self::init_partition`] does not populate. Peer shards observe the - /// seed through their shared left-right read handle, so only shard 0 is - /// seeded (a direct seed on a reader-mode peer STM would panic). The - /// simulator does not wire the reconciler, so this bypasses it the same - /// way `init_partition` bypasses it for the partition plane. Pair it with - /// `init_partition` for the same namespace on the dispatch-shell poll path. + /// Seed the metadata `Streams` STM on each live replica's shard-0 writer, so a + /// poll's `resolve_partition_namespace` succeeds for `namespace`, which the + /// partition-plane-only [`Self::init_partition`] does not populate. + /// + /// Shard 0 only: peers observe the seed through the shared left-right read + /// handle, and seeding a reader-mode peer STM directly would panic. The + /// reconciler is unwired here, so this bypasses it exactly as `init_partition` + /// does for the partition plane. Pair the two for the same namespace on the + /// dispatch-shell poll path. /// #[allow(clippy::cast_possible_truncation)] pub fn seed_stream_topic_partition(&self, namespace: IggyNamespace) { @@ -403,9 +509,8 @@ impl Simulator { if self.crashed.contains(&(i as u8)) { continue; } - // Shard 0 is the sole metadata writer; peers share its read handle - // and see the seed through the left-right publish, so seeding a - // peer's reader-mode STM directly would panic. + // Shard 0 is the sole metadata writer. Peers see the seed through the + // left-right publish, so seeding a reader-mode peer STM would panic. replica.shards[0] .plane .metadata() @@ -415,20 +520,35 @@ impl Simulator { } } - /// Log `client` in against the deterministic root user through the - /// dispatch shell: the real `on_client_request` path verifies the - /// seeded root credentials and runs the consensus `Register`, then this - /// binds the assigned session on the client. Requires the shell on and - /// the root user seeded (see [`new_shard`]); target is the primary - /// (replica 0), as [`Self::register_client_with_primary`] does. + /// Log `client` in against the deterministic root user through the dispatch + /// shell: the real `on_client_request` path verifies the seeded root credentials + /// and runs the consensus `Register`, then this binds the assigned session. + /// Needs the shell on and the root user seeded (see [`new_shard`]); targets the + /// primary, as [`Self::register_client_with_primary`] does. /// /// # Panics - /// If no login reply arrives within 200 steps or it carries no session. + /// If no login reply arrives within `SETUP_TOTAL_STEPS`, or it carries no + /// session. pub fn shell_login(&mut self, client: &SimClient) { - // Register the client's connection metadata on every replica, as - // `install_client_fd` does in production. `ensure_transport_connection` - // reads it to admit the connection into the SessionManager, which the - // login's session bind (Connected -> Authenticated -> Bound) requires. + self.shell_login_via(client, 0); + } + + /// [`Self::shell_login`] against a chosen replica. + /// + /// Dialing a BACKUP is the only way to reach register forwarding: the backup + /// verifies the credentials itself and sends only the consensus proposal on as + /// `ForwardRegister`, parking the login until the matching + /// `ForwardRegisterResult` returns, then answering on the connection it owns. A + /// client that always dials the primary never produces those frames. + /// + /// # Panics + /// If no login reply arrives within `SETUP_TOTAL_STEPS`, or it carries no + /// session. + pub fn shell_login_via(&mut self, client: &SimClient, target: u8) { + // Connection metadata on every replica, as `install_client_fd` does in + // production. `ensure_transport_connection` reads it to admit the connection + // into the SessionManager, which the login's bind + // (Connected, Authenticated, Bound) requires. let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0); for outbox in &self.outboxes { outbox.insert_client_meta(ClientConnMeta::new( @@ -438,40 +558,112 @@ impl Simulator { )); } - let msg = client.login(replica::SHELL_ROOT_USERNAME, replica::SHELL_ROOT_PASSWORD); - self.submit_request(client.client_id(), 0, msg.into_generic()); - let mut session = 0u64; - let mut got_reply = false; - for _ in 0..200 { - if let Some(reply) = self.step().first() { - // The login reply carries the assigned session in `op` - // (`build_reply_with_body` maps the session field to `op`). - session = reply.header().op; - got_reply = true; - break; - } - } - assert!(got_reply, "shell_login: no login reply within 200 steps"); + // From here this client talks the real client protocol. See + // `SimClient::shell_wire`. + client.set_shell_wire(); + let msg = client + .login(replica::SHELL_ROOT_USERNAME, replica::SHELL_ROOT_PASSWORD) + .into_generic(); + // `build_reply_with_body` maps the session field to `op`. + let session = self + .await_setup_reply( + client.client_id(), + target, + &msg, + iggy_binary_protocol::Operation::Register, + "shell_login", + ) + .header() + .op; assert!(session > 0, "shell_login: login reply carried no session"); client.bind_session(session); } + /// Submit `message` to `target` and step until a client reply arrives, + /// resubmitting every [`SETUP_RETRY_STEPS`] steps. + /// + /// Resubmitted verbatim, so the request id is stable. Not free, though: + /// `register_preflight` dispatches every `Register` past its gates, even one + /// whose client already holds an entry, because a bind is a fencing event and + /// absorbing it would hand back an un-bumped epoch. Each landed retry therefore + /// commits another register and fences the previous holder, recoverable in one + /// round trip but a reason to keep the interval well above it. + /// + /// # Panics + /// If no reply arrives within `SETUP_TOTAL_STEPS`. A handshake that never + /// completes leaves the fixture unusable, so there is no useful `None`. + fn await_setup_reply( + &mut self, + client_id: u128, + target: u8, + message: &Message, + expected_operation: iggy_binary_protocol::Operation, + label: &str, + ) -> Message { + let mut target = target; + for step in 0..SETUP_TOTAL_STEPS { + if step % SETUP_RETRY_STEPS == 0 { + self.submit_request(client_id, target, message.deep_copy()); + // Rotate, as the workload's resend path does. Retrying one replica + // forever suffices on a perfect network and is useless once it is + // partitioned or crashed mid-handshake: a register has to reach the + // metadata primary, and this one may be neither reachable nor able + // to forward. + target = (target + 1) % self.replica_count.max(1); + } + let mut answer = None; + for reply in self.step() { + let header = reply.header(); + // Correlate. Stepping the whole simulator surfaces every client's + // replies, and taking the first hands a two-client run another + // client's committed op as this answer, whose `op` is then bound as + // a session: a fenced client that evicts, recovers and repeats. + // + // Nor is a result-framed transport rejection an answer. Dispatch + // refused to place the request (not primary, transferring, queue + // full), stamped `op` with its commit point instead of a session and + // left `status` at 0, so it reads as a completed handshake and hands + // back a session the cluster never granted, silently, whenever that + // commit point is nonzero. + if answer.is_none() + && header.client == client_id + && header.operation == expected_operation + && !setup_reply_is_transient(&reply) + { + answer = Some(reply); + continue; + } + self.deferred_client_replies.push(reply); + } + if let Some(reply) = answer { + return reply; + } + } + panic!( + "{label}: no reply for client {client_id} within {SETUP_TOTAL_STEPS} steps \ + (seed {:#x})", + self.seed, + ); + } + /// Advance the simulation by one tick. Returns client replies delivered. /// - /// Every shard runs its real message pump as an executor task, so a - /// step is: fire the virtual consensus tick, let the pumps run to - /// quiescence, feed network packets into the shard routers, let the - /// pumps process the resulting frames, then exchange outboxes with the - /// network. Interleaving between pumps is a seeded executor pick; - /// everything a step produces lands on the wire before network time - /// advances, matching the pre-executor phase semantics. + /// Every shard runs its real message pump as an executor task. A step fires the + /// virtual consensus tick, runs the pumps to quiescence, feeds network packets + /// into the shard routers, runs the pumps over the resulting frames, then + /// exchanges outboxes with the network. Pump interleaving is a seeded executor + /// pick, and everything a step produces lands on the wire before network time + /// advances. /// /// # Panics - /// If a client-addressed packet cannot be decoded as `ReplyHeader`, or - /// if a pump livelocks (poll budget exhausted). + /// If a client-addressed packet does not decode as `ReplyHeader`, or a pump + /// livelocks (poll budget exhausted). #[allow(clippy::cast_possible_truncation)] pub fn step(&mut self) -> Vec> { - let mut client_replies = Vec::new(); + // Ahead of this step's own traffic, so a driver sees replies in delivery + // order rather than the order a handshake happened to interrupt. + let mut client_replies: Vec> = + std::mem::take(&mut self.deferred_client_replies); // Phase 0: Fire the pumps' consensus-tick timers (view change, // retransmits) and run them to quiescence. Crashed replicas have no @@ -489,22 +681,19 @@ impl Simulator { if !self.crashed.contains(&id) && let Some(replica) = self.replicas.get(id as usize) { - // Seeded homing: the receiving shard is usually NOT - // the owner, so the frame takes the real router hop - // (dispatch -> mesh -> owning pump), as it does when - // the coordinator homes a peer connection on an - // arbitrary shard. + // Seeded homing: the receiving shard is usually NOT the + // owner, so the frame takes the real router hop (dispatch, + // mesh, owning pump), as when the coordinator homes a peer + // connection on an arbitrary shard. let entry = self.entry_rng.random_range(0..replica.shards.len()); match packet.from { - // Shell mode: every client request enters through the - // real `on_client_request` handler (as the client-fd - // listener does in production), so it drains as a task - // the executor interleaves with the pump. Partition - // writes now use the same legacy `SendMessages` wire - // shape the real SDK sends, so + // Shell mode: client requests enter through the real + // `on_client_request` handler, as the client-fd listener + // does in production, and drain as a task the executor + // interleaves with the pump. Partition writes carry the + // legacy `SendMessages` shape the real SDK sends, so // `resolve_partition_request_namespace` decodes them - // on this path. Consensus frames (replica-sourced) - // always route raw. + // here. Replica-sourced consensus frames route raw. ProcessId::Client(client_id) if self.shell => { replica.shards[entry] .deliver_client_request(client_id, packet.message.deep_copy()); @@ -514,7 +703,16 @@ impl Simulator { } // Crashed or missing: packet silently dropped. } - ProcessId::Client(_) => { + ProcessId::Client(client_id) => { + // Not every client-addressed frame is a reply. `Eviction` tells + // a client its session is gone, sent once the client table drops + // it, which the dispatch shell reaches as soon as replicas crash + // and restart. Decoding it as a reply fails on the command + // discriminant, so classify first and record it for the driver. + if packet.message.header().command == Command::Eviction { + self.evicted.push(client_id); + continue; + } let reply: Message = packet .message .deep_copy() @@ -559,9 +757,88 @@ impl Simulator { client_replies } - /// Rolling hash of the executor schedule (every poll and timer fire). - /// Two runs from the same seed and inputs must agree; determinism - /// tests assert on it alongside the reply-trace hash. + /// Stamp a metadata snapshot watermark on one replica, standing in for a + /// checkpoint that superseded everything at or below `op`. + /// + /// Without a data directory there is no `SnapshotCoordinator`, so + /// `checkpoint_if_needed` returns immediately and the watermark stays at zero. + /// Nothing is evictable, the repair server has no compacted prefix to skip, and + /// `RangeEvicted`, the only signal converting a repair into a state transfer, + /// cannot occur however the cluster is driven. + /// + /// Stamping the number alone reaches the whole escalation without snapshot bytes + /// to transfer, so the protocol path is coverable without the coordinator seam a + /// real installing transfer needs. + /// + /// # Panics + /// If `replica_idx` is out of range. + pub fn stamp_metadata_snapshot(&self, replica_idx: usize, op: u64) { + use journal::Journal; + + self.replicas[replica_idx] + .metadata_journal + .set_snapshot_op(op); + } + + /// Submit this client's handshake to `target` WITHOUT stepping. + /// + /// The blocking helpers ([`Self::shell_login_via`], + /// [`Self::register_client_with_primary`]) are for fixture setup, before a driver + /// exists. Mid-run they step the simulator up to `SETUP_TOTAL_STEPS` times inside + /// the driver's tick, with no `Workload::tick`, no fault injection and no + /// invariant check, which is what `run_with_faults` is for. A driver recovering an + /// evicted client submits here and picks the reply up from its normal loop. + /// + /// Sends the handshake this simulator's mode can answer: a login on the shell, a + /// bare register on the raw path. + pub fn submit_handshake(&mut self, client: &SimClient, target: u8) { + let message = if self.shell { + client.set_shell_wire(); + client.login(replica::SHELL_ROOT_USERNAME, replica::SHELL_ROOT_PASSWORD) + } else { + client.register() + }; + self.submit_request(client.client_id(), target, message.into_generic()); + } + + /// The session a handshake reply carries, or `None` if it carries none. Not the + /// caller's business because the field differs by path: the shell's login answers + /// in `op`, the raw register in `commit`. + #[must_use] + pub fn handshake_session(&self, reply: &Message) -> Option { + if setup_reply_is_transient(reply) { + return None; + } + let header = reply.header(); + let session = if self.shell { header.op } else { header.commit }; + (session > 0).then_some(session) + } + + /// Have a rebuilt partition recover `(sequencer, commit, checksum)` from the log + /// this harness carried across the restart. + /// + /// Off by default, deliberately: the partition journal is in-memory and segments + /// carry no op numbers, so a real replica cannot do this and instead boots + /// quorum-invisible and asks the view's primary. Turn it on only to look past the + /// empty-frontier restart, which trips `advance_commit_min`'s sequential-advance + /// assert, at something later in the run; the run then tests a durability + /// guarantee production does not offer. + pub const fn set_restore_partition_frontier(&mut self, restore: bool) { + self.restore_partition_frontier = restore; + } + + /// Take the clients evicted since the last call. + /// + /// A driver must consume these: the session is gone, so outstanding requests are + /// unanswerable and the next one is refused until the client logs in again. + /// Ignoring them looks exactly like a wedge. + pub fn take_evictions(&mut self) -> Vec { + std::mem::take(&mut self.evicted) + } + + /// Rolling hash of the executor schedule: every poll and timer fire. Two runs + /// from the same seed and inputs must agree; determinism tests assert on it + /// alongside the reply-trace hash. #[must_use] pub const fn schedule_hash(&self) -> u64 { self.executor.schedule_hash() @@ -570,11 +847,10 @@ impl Simulator { /// Run the executor until every pump is parked again. /// /// # Panics - /// On budget exhaustion: pumps are event-driven, so this is a - /// spin-waking task, i.e. a livelock bug. The seed reproduces it. - /// - /// Also on a lost wakeup: a non-crashed pump quiescing with a non-empty - /// inbox (see [`Self::assert_inboxes_drained`]). + /// On budget exhaustion: pumps are event-driven, so it means a spin-waking task, + /// a livelock bug, reproducible from the seed. Also on a lost wakeup, a + /// non-crashed pump quiescing with a non-empty inbox (see + /// [`Self::assert_inboxes_drained`]). fn run_pumps(&mut self) { match self.executor.run_until_stalled(POLL_BUDGET) { RunOutcome::Quiescent { .. } => self.assert_inboxes_drained(), @@ -587,22 +863,20 @@ impl Simulator { } } - /// Lost-wake tripwire. At executor quiescence every live pump must have - /// drained its inbox: a non-empty inbox on a non-crashed replica means a - /// frame reached the channel without waking the target pump. Because - /// every pump holds a standing `CONSENSUS_TICK_INTERVAL` timer, the next - /// `advance_time` would re-poll and silently drain it, masking the exact - /// wake-loss class this harness exists to catch, so trip here instead. + /// Lost-wake tripwire. At executor quiescence every live pump must have drained + /// its inbox; a non-empty one on a non-crashed replica means a frame reached the + /// channel without waking the target pump. Every pump holds a standing + /// `CONSENSUS_TICK_INTERVAL` timer, so the next `advance_time` would re-poll and + /// silently drain it, masking the exact wake-loss class this harness exists to + /// catch. Trip here instead. /// - /// Incomplete by construction: it catches a lost wakeup only while the - /// un-woken frame is still queued at quiescence. A later frame that does - /// wake the pump drains the whole inbox (the recv loop pulls every queued - /// frame), so a lost wake masked by a subsequent drain slips through. The - /// direction is safe: a non-empty inbox at true quiescence is always a - /// real lost wake, so it never false-trips. + /// Incomplete by construction: only catches a lost wakeup while the un-woken + /// frame is still queued at quiescence, since a later frame that does wake the + /// pump drains the whole inbox. Safe in direction, though: a non-empty inbox at + /// true quiescence is always a real lost wake, so it never false-trips. /// - /// Crashed replicas are skipped: their pump tasks are aborted, so any - /// frame stranded in their inbox has no drainer and is expected. + /// Crashed replicas are skipped: their pump tasks are aborted, so a stranded + /// frame has no drainer and is expected. #[allow(clippy::cast_possible_truncation)] fn assert_inboxes_drained(&self) { for (replica_id, replica) in self.replicas.iter().enumerate() { @@ -634,8 +908,8 @@ impl Simulator { } } - /// Submit a client request into the simulated network. Equivalent to a - /// client opening a TCP connection and sending a message to a replica. + /// Submit a client request into the simulated network: a client opening a TCP + /// connection and sending a message to a replica. pub fn submit_request( &mut self, client_id: u128, @@ -649,55 +923,64 @@ impl Simulator { ); } - /// Register a client via the primary (replica 0). Sends `Register` - /// through the metadata plane and binds the assigned session on - /// `SimClient`. + /// Whether client requests go through the real dispatch shell. A driver has to + /// know: the paths do not share a handshake (`Register` versus a login that mints + /// a session), so re-establishing a client mid-run has to pick the served one. + #[must_use] + pub const fn is_shell(&self) -> bool { + self.shell + } + + /// Register a client via the primary (replica 0): sends `Register` through the + /// metadata plane and binds the assigned session on `SimClient`. /// /// # Panics - /// If no reply arrives within 100 steps. - #[allow(clippy::cast_possible_truncation)] + /// If no reply arrives within `SETUP_TOTAL_STEPS`. pub fn register_client_with_primary(&mut self, client: &SimClient) { - let msg = client.register(); - self.submit_request(client.client_id(), 0, msg.into_generic()); - let mut session = 0u64; - let mut got_reply = false; - for _ in 0..100 { - let replies = self.step(); - if !replies.is_empty() { - let header = replies[0].header(); - debug_assert_eq!( - header.operation, - iggy_binary_protocol::Operation::Register, - "register_client_with_primary: first reply was not Register" - ); - assert_eq!( - header.client, - client.client_id(), - "register_client_with_primary: reply client_id mismatch \ - (expected {}, got {})", - client.client_id(), - header.client, - ); - session = header.commit; - got_reply = true; - break; - } - } - assert!( - got_reply, - "register_client_with_primary: no reply within 100 steps" + self.register_client_via(client, 0); + } + + /// [`Self::register_client_with_primary`] against a chosen replica, the raw + /// counterpart of [`Self::shell_login_via`]. A driver re-registering an evicted + /// client picks a live replica, and the primary may be the one whose restart + /// caused the eviction. + /// + /// # Panics + /// If no reply arrives within `SETUP_TOTAL_STEPS`. + #[allow(clippy::cast_possible_truncation)] + pub fn register_client_via(&mut self, client: &SimClient, target: u8) { + let msg = client.register().into_generic(); + let reply = self.await_setup_reply( + client.client_id(), + target, + &msg, + iggy_binary_protocol::Operation::Register, + "register_client_with_primary", + ); + let header = reply.header(); + debug_assert_eq!( + header.operation, + iggy_binary_protocol::Operation::Register, + "register_client_with_primary: first reply was not Register" ); - client.bind_session(session); + assert_eq!( + header.client, + client.client_id(), + "register_client_with_primary: reply client_id mismatch (expected {}, got {})", + client.client_id(), + header.client, + ); + client.bind_session(header.commit); - // Partition has no `client_table`: at-least-once, no per-client - // dedup. Consumers dedup via message id / content / producer-id+seq. - // Sessions/dedup/eviction live on metadata only (IggyMetadata). + // Partitions have no `client_table`: at-least-once, no per-client dedup, so + // consumers dedup on message id, content or producer-id+seq. Sessions, + // dedup and eviction live on metadata only. } - /// Crash a replica: abort its pump tasks, disable its network links, and discard - /// its outbox. The replica object stays alive but receives no messages; a - /// following [`Self::replica_restart`] drops and rebuilds it from the durable - /// superblock, which is when volatile state is actually lost and recovered. + /// Crash a replica: abort its pump tasks, disable its network links, discard its + /// outbox. The object stays alive but receives nothing; a following + /// [`Self::replica_restart`] drops and rebuilds it from the durable superblock, + /// which is where volatile state is actually lost and recovered. /// /// # Panics /// If the replica is already crashed. @@ -707,16 +990,15 @@ impl Simulator { "cannot crash replica {replica_index}: already down" ); - // Hard-stop the pumps: futures drop mid-await, destructors cancel - // their channel and timer registrations, and the graceful inbox - // drain never runs: a crash, not a shutdown. + // Hard stop: futures drop mid-await, destructors cancel their channel and + // timer registrations, and the graceful inbox drain never runs. A crash, not + // a shutdown. for task in &self.replicas[replica_index as usize].pump_tasks { self.executor.abort(*task); } - // Tear down any detached dispatch tasks this replica's bus spawned - // (off-pump poll IO, request drains), so a crash leaves no orphaned - // tasks running against the dead replica. + // Detached dispatch tasks this replica's bus spawned (off-pump poll IO, + // request drains), so a crash leaves none running against a dead replica. self.executor.abort_replica_spawned(replica_index); // Discard any unsent messages (never reached the wire). @@ -736,15 +1018,15 @@ impl Simulator { } /// Restart a crashed replica: drop its shards, losing all volatile consensus - /// state as a real restart does, and rebuild them against the retained - /// superblock, recovering `(view, log_view)` from disk exactly as production's - /// `restore_metadata_consensus` does. The superblock and outbox are - /// harness-owned, so they survive the drop; a fresh inter-shard mesh and pump - /// tasks are wired, and the network is re-enabled. + /// state as a real restart does, and rebuild against the retained superblock, + /// recovering `(view, log_view)` from disk as production's + /// `restore_metadata_consensus` does. Superblock and outbox are harness-owned and + /// survive the drop; a fresh mesh and pump tasks are wired and the network + /// re-enabled. /// /// # Panics - /// If the replica is not crashed, or if its shard count does not fit `u16`, - /// impossible since mesh construction caps it. + /// If the replica is not crashed, or its shard count does not fit `u16`; mesh + /// construction caps it. pub fn replica_restart(&mut self, replica_index: u8) { assert!( self.crashed.contains(&replica_index), @@ -754,26 +1036,33 @@ impl Simulator { let shards_per_replica = u16::try_from(self.replicas[idx].shards.len()).expect("shard count fits u16"); let superblock = Rc::clone(&self.replicas[idx].superblock); - // The metadata WAL is harness-owned too, so its bytes and index survive the - // drop: the rebuilt shard 0 recovers op/commit and committed state from it, - // not from an empty journal. + // The metadata WAL is harness-owned too, so the rebuilt shard 0 recovers + // op/commit and committed state from it rather than from an empty journal. let metadata_journal = Rc::clone(&self.replicas[idx].metadata_journal); - // Bump the incarnation on restart, so a StartView addressed to the previous - // incarnation and still in flight is ignored. Deterministic, so replay stays - // byte-identical. + // Bumped so an in-flight StartView addressed to the previous incarnation is + // ignored. Deterministic, so replay stays byte-identical. let metadata_incarnation = self.replicas[idx].metadata_incarnation + 1; - // Partition superblocks carry forward too: a group re-materialised after - // the restart must recover its recorded view from the same store, exactly - // as a rebooted server partition reads the record in its directory. + // Partition superblocks carry forward too: a re-materialised group must + // recover its recorded view from the same store, as a rebooted server + // partition reads the record in its directory. let partition_superblocks = std::mem::take(&mut *self.replicas[idx].partition_superblocks.borrow_mut()); - - // Recover the durable VSR state from the retained superblock before the - // rebuild, as production reads it in restore_metadata_consensus. + // Take each live partition's log while its shard still stands: a real + // server's messages are in segment files its boot recovers the offset counter + // from, so rebuilding with nothing would model total data loss rather than a + // restart. Before the rebuild, which drops the shards. + let partition_logs = self.retain_partition_logs(idx, &partition_superblocks); + + // Durable VSR state from the retained superblock, before the rebuild, as + // production reads it in `restore_metadata_consensus`. let recovered_state = superblock .read_latest_sync() .and_then(|bytes| VsrState::try_from(bytes.as_slice()).ok()); + // Carried like the WAL and superblocks: the rebuilt replica has to find the + // snapshot its previous incarnation persisted. + let replica_data_dir = self.replicas[idx].data_dir.clone(); + let consensus_clock = ConsensusClock::new(Rc::new(SimClock::new(self.executor.timer()))); let outbox = Rc::clone(&self.outboxes[idx]); let (senders, mut inboxes, mut reply_inboxes) = @@ -812,6 +1101,7 @@ impl Simulator { shard_journal, recovered_state, metadata_incarnation, + (shard_idx == 0).then(|| replica_data_dir.clone()).flatten(), ); if shard_idx == 0 { metadata_bundle = @@ -827,25 +1117,26 @@ impl Simulator { } // Replacing the replica drops the old shards, losing all volatile consensus - // state as a real restart does. The harness-owned superblock carries forward. + // state as a real restart does. The harness-owned superblock carries over. self.replicas[idx] = SimReplica { shards, superblock, metadata_journal, metadata_incarnation, partition_superblocks: RefCell::new(partition_superblocks), + partition_logs: RefCell::new(partition_logs), + data_dir: replica_data_dir, _stop_txs: stop_txs, pump_tasks, }; // Re-materialise every group this replica had before the crash, as a - // rebooted the server re-opens every partition directory it owns. This - // is what makes the carried-forward superblock load-bearing: the group - // recovers the `(view, log_view)` it recorded instead of re-entering - // view 0. - // SORTED: `HashMap` iteration order is seeded per process, and - // materialisation order is observable (shard init order, routing-row - // stamps), so replay would stop being byte-identical. + // rebooted server re-opens every partition directory it owns. This is what + // makes the carried-forward superblock load-bearing: the group recovers its + // recorded `(view, log_view)` instead of re-entering view 0. + // SORTED: `HashMap` order is seeded per process and materialisation order is + // observable (shard init order, routing-row stamps), so replay would stop + // being byte-identical. let mut materialised: Vec = self.replicas[idx] .partition_superblocks .borrow() @@ -854,7 +1145,11 @@ impl Simulator { .collect(); materialised.sort_unstable_by_key(IggyNamespace::inner); for namespace in materialised { - materialise_partition(&self.replicas[idx], namespace); + materialise_partition( + &self.replicas[idx], + namespace, + self.restore_partition_frontier, + ); } // Reconnect to the network and mark the replica live again. @@ -863,9 +1158,46 @@ impl Simulator { self.crashed.remove(&replica_index); } - /// Advance consensus timeouts on every live replica without a full - /// step cycle: fires the pumps' virtual tick timers and runs the - /// executor to quiescence. + /// Take the message log out of every materialised partition, with the offsets + /// recovered from it. + /// + /// Called while the outgoing shards are still alive, the last point the data can + /// be read. `std::mem::take` leaves an empty log behind, which nothing observes: + /// that shard is dropped moments later. + /// + /// Keyed off `partition_superblocks`, already the record of which groups this + /// replica materialised and what the restart re-materialises from. + fn retain_partition_logs( + &self, + replica_idx: usize, + materialised: &HashMap>, + ) -> HashMap { + let replica = &self.replicas[replica_idx]; + let mut retained = HashMap::with_capacity(materialised.len()); + for &namespace in materialised.keys() { + let partitions = replica.partition_shard(namespace).plane.partitions(); + let Some(partition) = partitions.get_mut_by_ns(&namespace) else { + continue; + }; + let offsets = partition.offsets(); + // `offset_space_used` off the RETIRING partition: an untouched one and + // one holding a single message at offset 0 both report `(0, 0)`, and only + // this instance still knows which it is. + retained.insert( + namespace, + RetainedPartitionState { + log: std::mem::take(&mut partition.log), + durable_offset: offsets.commit_offset, + write_offset: offsets.write_offset, + offset_space_used: partition.offset_space_used(), + }, + ); + } + retained + } + + /// Advance consensus timeouts on every live replica without a full step cycle: + /// fire the pumps' virtual tick timers, run the executor to quiescence. /// /// # Panics /// If a pump livelocks (poll budget exhausted). @@ -887,19 +1219,16 @@ impl Simulator { ) -> Result, IggyError> { let shard = self.replicas[replica_idx].partition_shard(namespace); // Build the owned poll plan synchronously, then execute off the borrow. - // The sim's partitions are in-memory (no `partition_dir`), so the plan - // serves only the resident journal tier; `execute` performs no disk IO. // - // This is the one `block_on` allowed to stay, and only because - // `plan.execute()` cannot suspend here: the sim's partitions are - // in-memory, so the plan serves the resident journal tier with no disk - // IO and no `bus.sleep`. If it ever grew a suspending await it would - // fail two ways -- on the virtual clock it would hang this thread - // forever (the clock only advances through `advance_time`, which does - // not run during `block_on`), and on the retry path it would panic on - // the compio timer outside a compio runtime. Safe today only because - // it runs between `run_pumps` calls, when the executor is quiescent and - // no pump can hold the partition commit lock in a suspended frame. + // The one `block_on` allowed to stay, and only because `plan.execute()` + // cannot suspend here: the sim's partitions are in-memory (no + // `partition_dir`), so the plan serves the resident journal tier with no disk + // IO and no `bus.sleep`. A suspending await would fail two ways. On the + // virtual clock it would hang this thread forever, the clock advancing only + // through `advance_time`, which does not run during `block_on`. On the retry + // path it would panic on the compio timer outside a compio runtime. Safe only + // because it runs between `run_pumps` calls, with the executor quiescent and + // no pump holding the partition commit lock in a suspended frame. let Some(plan) = shard .plane .partitions() @@ -909,9 +1238,8 @@ impl Simulator { "partition not found for namespace {namespace:?} on replica {replica_idx}" ))); }; - // The simulator drives partitions directly, so it never replicates a - // poll's auto-commit (that is the serving shard's job in the real - // server); the surfaced offset is discarded here. + // Partitions are driven directly, so a poll's auto-commit is never + // replicated (the serving shard's job in the real server). Offset discarded. let (fragments, _commit_offset, _auto_commit) = futures::executor::block_on(plan.execute()); Ok(fragments) } @@ -928,8 +1256,8 @@ impl Simulator { Some(partition.offsets()) } - /// Consensus view for a replica's partition-plane group, or `None` if the - /// namespace is not present on that replica. + /// Consensus view for a replica's partition-plane group, or `None` if that + /// replica does not host the namespace. #[must_use] pub(crate) fn consensus_view( &self, @@ -941,15 +1269,35 @@ impl Simulator { Some(u64::from(partition.consensus().view())) } + /// One replica's view of a partition group's consensus, or `None` when it does + /// not host the namespace. Read by the quiesce oracle to decide whether a group + /// has settled into one view, which its leader-relative checks depend on once + /// partition primaries can be crashed. + #[must_use] + pub(crate) fn partition_consensus_state( + &self, + replica_idx: usize, + namespace: IggyNamespace, + ) -> Option { + let shard = self.replicas[replica_idx].partition_shard(namespace); + let partition = shard.plane.partitions().get_by_ns(&namespace)?; + let consensus = partition.consensus(); + Some(PartitionConsensusState { + status: consensus.status(), + view: consensus.view(), + is_primary: consensus.is_primary(), + commit_min: consensus.commit_min(), + }) + } + /// Index of the current primary for `namespace`, as seen by the first live /// replica hosting it, or `None` if no live replica hosts it. /// - /// Reads one replica's view, so it assumes live replicas agree on the - /// primary. Sound only while the primary is static, which holds today: the - /// driver spares primaries from crashes, so no crash-triggered view change - /// runs mid-test. Once primary-crash injection lands, views can diverge and - /// this may name a stale or crashed primary (see - /// `workload::oracle::assert_converged` for the consequence and fix). + /// Reads one replica's view, so it assumes live replicas agree on the primary. + /// That no longer holds by construction: `spare_primary` is off under + /// `--crash-primary`, so a crash-triggered view change can run mid-run and this + /// may name a stale or crashed primary. Callers needing a real answer run + /// `workload::oracle::settle_to_stable_view` first. #[must_use] pub(crate) fn primary_index(&self, namespace: IggyNamespace) -> Option { (0..self.replica_count) @@ -969,15 +1317,30 @@ impl Simulator { /// Materialises `namespace` on its hash-owning shard of one replica and stamps /// the routing row on every shard of that replica. /// -/// Shared by [`SimCluster::init_partition`] and the restart path: a rebooted -/// server re-opens every partition directory it owns, so the sim has to -/// re-materialise too, otherwise the superblock a restart carries forward is -/// never read back and the recovered-view branch is dead code. -fn materialise_partition(replica: &SimReplica, namespace: IggyNamespace) { +/// Shared by [`SimCluster::init_partition`] and the restart path: a rebooted server +/// re-opens every partition directory it owns, so the sim must re-materialise too, +/// else the superblock a restart carries forward is never read back and the +/// recovered-view branch is dead code. +fn materialise_partition(replica: &SimReplica, namespace: IggyNamespace, restore_frontier: bool) { let shard_count = u32::try_from(replica.shards.len()).expect("shard count fits u32"); let owner = calculate_shard_assignment(&namespace, shard_count); - // One store per group, minted on first materialisation and reused on every - // later one, so the recorded view survives a replica restart. + // Commit the namespace first: a partition the metadata plane never heard of is + // a shape production cannot produce, and the shard refuses client traffic whose + // routing-row epoch it cannot match against a committed `created_revision`. + let streams = replica.shards[0].plane.metadata().mux_stm.streams(); + streams.seed_namespace(namespace, namespace.inner()); + // No committed revision means the seed could not re-add the namespace, which + // happens once a metadata workload has deleted its stream or topic: the seed's + // `CreatePartitions` is then a committed REJECTION rather than an error, so it + // reports nothing. Skip the group rather than build a partition no committed + // metadata names, as a rebooted server does not re-open a deleted partition's + // directory either. Before the build, so a skipped group leaves neither a + // partition nor a routing row behind. + let Some(epoch) = streams.created_revision_for_namespace(namespace) else { + return; + }; + // One store per group, minted on first materialisation and reused after, so the + // recorded view survives a replica restart. let superblock = Rc::clone( replica .partition_superblocks @@ -988,16 +1351,18 @@ fn materialise_partition(replica: &SimReplica, namespace: IggyNamespace) { let recovered_state = superblock .read_latest_sync() .and_then(|bytes| VsrState::try_from(bytes.as_slice()).ok()); - replica.shards[usize::from(owner)].init_partition(namespace, Some(superblock), recovered_state); - // Commit the namespace before stamping the rows: a partition the metadata - // plane never heard of is a shape production cannot produce, and the shard - // refuses to serve client traffic whose routing-row epoch it cannot match - // against a committed `created_revision`. - let streams = replica.shards[0].plane.metadata().mux_stm.streams(); - streams.seed_namespace(namespace, namespace.inner()); - let epoch = streams - .created_revision_for_namespace(namespace) - .expect("namespace committed by the seed above"); + // Hand back the log this group left behind, if it was materialised here before. + // Removed rather than cloned: the rebuilt partition becomes its sole owner, and + // a second materialisation with no restart between would otherwise resurrect a + // log the live partition has moved past. + let retained = replica.partition_logs.borrow_mut().remove(&namespace); + replica.shards[usize::from(owner)].init_partition( + namespace, + Some(superblock), + recovered_state, + retained, + restore_frontier, + ); for shard in &replica.shards { shard.shards_table().insert( namespace, @@ -1111,10 +1476,10 @@ mod tests { ); } - /// A metadata replica that advanced its view, persisted it through the superblock - /// gate, then crashed recovers that same view from its own disk on restart, not a - /// fresh 0. The split-brain guarantee: a replica never forgets a view it acted in. - /// Impossible before the superblock, since a rebuilt consensus starts at view 0. + /// A replica that advanced its view, persisted it through the superblock gate, + /// then crashed recovers that view from its own disk, not a fresh 0. The + /// split-brain guarantee: a replica never forgets a view it acted in. Impossible + /// before the superblock, a rebuilt consensus starting at view 0. #[test] fn given_advanced_view_when_metadata_replica_restarts_should_recover_view_from_superblock() { server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { @@ -1186,9 +1551,9 @@ mod tests { "restarted replica must recover its persisted view from the superblock, not reset to 0" ); - // This run takes zero traffic, so every WAL is empty: the recovered view came - // from the superblock alone. Pin that, since it is what makes the restart gate - // below load-bearing. + // Zero traffic, so every WAL is empty and the recovered view came from the + // superblock alone. Pinned, since it is what makes the gate below + // load-bearing. assert!( sim.replicas[primary as usize] .metadata_journal @@ -1197,10 +1562,10 @@ mod tests { "no metadata traffic in this test, so the restarted replica's WAL must be empty" ); - // A recovered view is a prior life even with an empty WAL, so the replica must - // rejoin as a probing backup. It is still primary-by-index for the recovered - // view, so resuming primaryship here would have it act as primary in a view the - // survivors may already have left, with no probe to correct it. + // A recovered view is a prior life even with an empty WAL, so the replica + // rejoins as a probing backup. Still primary-by-index for that view, so + // resuming primaryship would have it act as primary in a view the survivors + // may already have left, with no probe to correct it. let restarted = sim.replicas[primary as usize].shards[0] .plane .metadata() @@ -1225,173 +1590,18 @@ mod tests { ); } - #[test] - fn given_committed_metadata_when_solo_replica_restarts_should_recover_from_own_wal() { - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: 1, - client_count: 1, - ..packet::PacketSimulatorOptions::default() - }; - // Solo cluster: 1-of-1 quorum commits every metadata op the instant it is - // journaled, giving a fully-committed WAL with no uncommitted suffix to - // reconcile on restart. - let mut sim = Simulator::new(1, std::iter::once(client_id), network_opts); - let client = SimClient::new(client_id); - sim.register_client_with_primary(&client); - - // Resolves the namespace of the stream and topic created below; `Some` exactly - // when the Streams STM holds them. - let resolve = |sim: &Simulator| { - sim.replicas[0].shards[0] - .plane - .metadata() - .mux_stm - .streams() - .namespace_from_partition( - &iggy_binary_protocol::WireIdentifier::named("events").unwrap(), - &iggy_binary_protocol::WireIdentifier::named("logs").unwrap(), - 0, - ) - }; - - // Drive committed metadata through consensus: a stream, then a topic with one - // partition under it. Each appends a prepare to shard 0's WAL and mutates the - // Streams STM. The topic references the stream, so the stream commits first. - for msg in [ - client.create_stream("events"), - client.create_topic("events", "logs", 1), - ] { - sim.submit_request(client_id, 0, msg.into_generic()); - for _ in 0..50 { - sim.step(); - } - } - - let namespace_before = resolve(&sim).expect("stream + topic must resolve after creation"); - let head_before = sim.replicas[0] - .metadata_journal - .last_op() - .expect("metadata ops must have been appended to the WAL"); - let commit_before = sim.replicas[0].shards[0] - .plane - .metadata() - .consensus - .as_ref() - .expect("solo shard 0 owns metadata consensus") - .commit_min(); - assert_eq!( - commit_before, head_before, - "a solo replica commits every durable op, so commit tracks the WAL head" - ); - - // Crash and restart: the shards are dropped, losing all volatile consensus and - // state-machine state, and rebuilt against the RETAINED WAL and superblock, so - // recovery is from this replica's own disk with no peer. - sim.replica_crash(0); - sim.replica_restart(0); - - // The WAL bytes and index survived the restart. - assert_eq!( - sim.replicas[0].metadata_journal.last_op(), - Some(head_before), - "the metadata WAL head must survive a restart, bytes and index retained" - ); - // Consensus recovered its op/commit from its own disk, not a fresh 0. - let consensus_ref = sim.replicas[0].shards[0].plane.metadata(); - let consensus = consensus_ref - .consensus - .as_ref() - .expect("restarted solo shard 0 owns metadata consensus"); - assert_eq!( - consensus.commit_min(), - head_before, - "commit must be recovered from the retained WAL, not reset to 0" - ); - // Replaying the retained WAL reconstructed the committed Streams STM, so the - // stream and topic survive the restart from this replica's own disk. - assert_eq!( - resolve(&sim), - Some(namespace_before), - "the created stream/topic must survive the restart via WAL replay" - ); - } - - #[test] - fn given_registered_client_when_solo_replica_restarts_should_recover_session_from_own_wal() { - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: 1, - client_count: 1, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new(1, std::iter::once(client_id), network_opts); - let client = SimClient::new(client_id); - - // Register creates the client-table session; one committed metadata op caches - // a reply for at-most-once dedup. Both live in the client table, which is not - // part of the state machine and would otherwise reset to empty on restart. - sim.register_client_with_primary(&client); - sim.submit_request(client_id, 0, client.create_stream("events").into_generic()); - for _ in 0..50 { - sim.step(); - } - - let epoch_before = sim.replicas[0].shards[0] - .plane - .metadata() - .client_table - .borrow() - .get_epoch(client_id); - assert!( - epoch_before.is_some(), - "client must hold a session before the crash" - ); - - // Crash and restart: the client table drops with the shard and is rebuilt by - // replaying the retained WAL through the same commit apply path the live - // cluster uses. - sim.replica_crash(0); - sim.replica_restart(0); - - let epoch_after = sim.replicas[0].shards[0] - .plane - .metadata() - .client_table - .borrow() - .get_epoch(client_id); - assert_eq!( - epoch_after, epoch_before, - "the client session must survive a restart, reconstructed from the retained WAL, \ - so a returning client is recognized instead of hitting NoSession" - ); - } - #[test] fn given_superblock_write_fails_when_primary_crashes_should_withhold_votes_and_not_elect() { - // The split-brain gate under test: a view-scoped send (SVC, DVC, StartView) - // must not go out until the new view is durable. Fail one survivor's - // superblock writes so its persist gate returns false, advancing its view - // in-memory while withholding every view-scoped send. In a 3-replica cluster, - // quorum 2, crashing the primary leaves one working survivor whose lone vote - // cannot reach quorum, so NO new primary is elected. Were the gate to send - // before persisting, the withheld votes would reach the peer and elect a - // primary that has no durable record of the new view, reintroducing - // split-brain on its restart. The positive control is - // `given_advanced_view_when_metadata_replica_restarts_...`, which elects a new - // primary from the same crash with healthy superblocks. + // The gate under test: a view-scoped send (SVC, DVC, StartView) must not go + // out until the new view is durable. Failing one survivor's superblock writes + // makes its persist gate return false, advancing its view in-memory while + // withholding every view-scoped send. Quorum 2 of 3, so crashing the primary + // leaves one working survivor whose lone vote cannot reach quorum and NO new + // primary is elected. Sending before persisting would instead elect a primary + // with no durable record of the new view, reintroducing split-brain on its + // restart. Positive control: + // `given_advanced_view_when_metadata_replica_restarts_...`, same crash with + // healthy superblocks. server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), @@ -1411,9 +1621,9 @@ mod tests { network_opts, ); - // Replica 1 survives the crash and is the primary-by-index for view 1. - // Failing its superblock keeps it from durably taking any new view, so it - // never emits a view-scoped vote. + // Replica 1 survives and is primary-by-index for view 1. Failing its + // superblock keeps it from durably taking any new view, so it never emits a + // view-scoped vote. sim.replicas[1].superblock.set_fail_writes(); sim.replica_crash(0); @@ -1459,10 +1669,10 @@ mod tests { "a failing superblock persists nothing, so no new view is durable" ); - // The retry is bounded. Without a backoff the 10 ms consensus tick would run a - // full `atomic_replace` (create, write, fsync, rename, dir fsync) on every tick - // for as long as the disk stays broken, on the executor that also serves - // partition traffic. + // Bounded retry. Without a backoff the 10 ms consensus tick would run a full + // `atomic_replace` (create, write, fsync, rename, dir fsync) every tick for as + // long as the disk stays broken, on the executor that serves partition + // traffic too. let attempts = sim.replicas[1].shards[0] .plane .metadata() @@ -1475,10 +1685,10 @@ mod tests { ); } - /// At-least-once failover: `SendMessages` retry on a new primary - /// re-executes. Retry reply carries a HIGHER `commit` op (re-execution - /// proof, not dedup). Duplicate payload lives at two offsets; consumers - /// dedup if they need at-most-once-per-payload. + /// At-least-once failover: a `SendMessages` retry on a new primary re-executes. + /// The retry reply carries a HIGHER `commit` op, proof of re-execution rather + /// than dedup, and the duplicate payload lives at two offsets. Consumers dedup + /// if they want at-most-once-per-payload. #[test] fn failover_retry_re_executes_under_at_least_once() { server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { @@ -1556,7 +1766,7 @@ mod tests { "new primary must not be the crashed replica" ); - // Replay SAME request to new primary. No dedup -> re-execution. + // Replay the SAME request to the new primary. No dedup, so re-execution. sim.submit_request(client_id, new_primary_idx, replay_req.into_generic()); let mut retry_reply: Option> = None; @@ -1591,90 +1801,21 @@ mod tests { ); } - /// Regression: a behind backup (`commit_min < commit_max`) becoming - /// primary must not panic during the `CommitMessage` heartbeat timeout. - /// `handle_commit_message_timeout` used to assert `commit_min == commit_max`. + /// Determinism: fresh simulator + workload from the same seed (network + /// and workload) produces an identical reply-header sequence. #[test] - fn view_change_behind_backup_becomes_primary() { + fn workload_replay_is_deterministic() { server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), bucket_capacity: 1, }); - let replica_count: u8 = 3; - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, - client_count: 1, - ..packet::PacketSimulatorOptions::default() - }; - - let mut sim = Simulator::new( - replica_count as usize, - std::iter::once(client_id), - network_opts, - ); - let client = SimClient::new(client_id); - let ns = IggyNamespace::new(1, 1, 0); - sim.init_partition(ns); - - // Register the client with the consensus cluster. - sim.register_client_with_primary(&client); - - // Send several messages so primary commits ahead of backups. - // Backups receive prepares but may lag on commit (`commit_max` < - // primary's `commit_min`): commit point only propagates via later - // Prepare headers or Commit heartbeats. - for i in 0..3 { - let msg = client.send_messages(ns, &[Bytes::from(format!("msg-{i}"))]); - sim.submit_request(client_id, 0, msg.into_generic()); - // Few steps: enough for replication, not enough for backups - // to fully learn the commit point. - for _ in 0..10 { - sim.step(); - } - } - - // Crash the primary immediately. Backups may have commit_min < commit_max. - sim.replica_crash(0); - - // Run view change. This must not panic in handle_commit_message_timeout. - for _ in 0..800 { - sim.step(); - } - - // Verify a new primary was elected and is functional. - let mut new_primary_found = false; - for idx in 1..replica_count { - let c = sim.replicas[idx as usize].shards[0] - .plane - .partitions() - .get_by_ns(&ns) - .expect("partition must exist on every live replica") - .consensus(); - if c.view() > 0 && c.status() == Status::Normal && c.is_primary() { - new_primary_found = true; - } - } - assert!(new_primary_found, "expected a new primary"); - } - - /// Determinism: fresh simulator + workload from the same seed (network - /// and workload) produces an identical reply-header sequence. - #[test] - fn workload_replay_is_deterministic() { - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let h1 = workload_hash_for_seed(0xDEAD_BEEF); - let h2 = workload_hash_for_seed(0xDEAD_BEEF); - assert_eq!( - h1, h2, - "workload reply hash diverged across runs with the same seed" + let h1 = workload_hash_for_seed(0xDEAD_BEEF); + let h2 = workload_hash_for_seed(0xDEAD_BEEF); + assert_eq!( + h1, h2, + "workload reply hash diverged across runs with the same seed" ); // Sanity: a different seed should generally produce a different @@ -1686,241 +1827,37 @@ mod tests { ); // Fragile cross-run baseline, pinned to seed 0xDEAD_BEEF under the default - // `ActionWeights`. Drifts whenever reply shape, partition commit values, or - // PRNG draw order change. Draw order is sensitive to `pick_outcome`: adding - // an outcome to an op sampled in this seed's window, or a weight bump, - // shifts the trace. Re-lock on intentional changes; expect re-locks until - // error discriminants and reply bodies stabilize the wire format. + // `ActionWeights`. Drifts on any change to reply shape, partition commit + // values, or PRNG draw order. Draw order is sensitive to `pick_outcome`, so + // adding an outcome to an op sampled in this window, or bumping a weight, + // shifts the trace. Expect re-locks until error discriminants and reply bodies + // stabilize the wire format. // - // Re-locked when the sim adopted METADATA_GROUP (1<<63) - // for metadata requests and the metadata consensus group, replacing - // the sim-only 0: reply headers and the per-group timeout-jitter seed - // (replica_id ^ namespace) both changed. The old 0 only ever routed - // correctly because `hash % 1 == 0` at one shard per replica. - // Re-locked again when replies stopped echoing a group id (the - // client wire lost its namespace field): the reply-hash tuple - // dropped that component. Re-locked when the v1 consumer-offset ops - // were removed and the v2 pair became the only store/delete actions: - // `Action` lost two variants, shifting discriminants and draw order. + // Re-locked for: METADATA_GROUP (1<<63) replacing the sim-only 0, moving both + // reply headers and the `replica_id ^ namespace` jitter seed; replies dropping + // the group id from the client wire; the v1 consumer-offset ops being removed, + // shifting `Action` discriminants and draw order; those ops drawing the WIRE + // consumer kind (1 / 2) instead of a bare boolean, kind 0 being no + // `WireConsumer` discriminant so every such request was dropped unparsed (see + // `ops::sample_consumer_kind`); and per-stream seeds moving from XOR salts to + // [`SimSeeds`] alongside `Xoshiro256Plus` becoming `Xoshiro256PlusPlus`, which + // together remap every stream. assert_eq!( - h1, 0x14BF_3C3F_41D4_F69D, + h1, 0x1376_D480_4A3F_E6A9, "workload reply hash drifted from locked baseline" ); } - /// Drive workload with near-uniform weights across all 23 `Action` variants. - /// Assert it runs without panic and observes at least one reply. - /// Per-op coverage not asserted: some ops can starve the in-flight slot - /// at single-client / 1-slot pipeline limits. - #[test] - fn uniform_weights_runs_clean() { - use crate::workload::{ - Workload, - actions::Action, - options::{ActionWeights, WorkloadOptions}, - }; - use strum::{EnumCount, IntoEnumIterator}; - - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let replica_count: u8 = 3; - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, - client_count: 1, - seed: 0xC0FF_EE00, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new( - replica_count as usize, - std::iter::once(client_id), - network_opts, - ); - let client = client::SimClient::new(client_id); - let ns_a = server_common::sharding::IggyNamespace::new(1, 1, 0); - let ns_b = server_common::sharding::IggyNamespace::new(1, 1, 1); - sim.init_partition(ns_a); - sim.init_partition(ns_b); - sim.register_client_with_primary(&client); - - // 23 variants: 8 x 5 + 15 x 4 = 100 (weights must sum to 100). - assert_eq!(Action::COUNT, 23, "Action::COUNT changed; adjust weights"); - let entries: Vec<(Action, u8)> = Action::iter() - .enumerate() - .map(|(index, action)| (action, if index < 8 { 5 } else { 4 })) - .collect(); - let weights = ActionWeights::new(&entries); - - let mut options = WorkloadOptions::new(0xC0FF_EE00, replica_count, vec![ns_a, ns_b]); - options.weights = weights; - let mut wl = Workload::new(options); - - let mut replies_seen = 0u64; - for _tick in 0..2_000u32 { - if let Some((target, msg)) = wl.build_request(&client) { - sim.submit_request(client.client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let cmds = wl.on_reply(&reply); - apply_sim_commands(&mut sim, &cmds); - replies_seen += 1; - } - } - assert!( - replies_seen > 0, - "uniform-weight workload produced no replies; sampling or dispatch broken" - ); - } - - /// The cheap per-tick invariants run inside `workload::run` and - /// stay green over a uniform 25-op single-client workload. A second pass - /// with a fresh `Invariants` confirms the checks observed live state - /// (non-vacuous): `commit_offset` is tracked for every (replica, namespace) - /// pair, not silently skipped. - #[test] - fn uniform_weights_invariants_hold() { - use crate::workload::{ - self, Workload, - actions::Action, - invariants::Invariants, - options::{ActionWeights, WorkloadOptions}, - }; - use strum::{EnumCount, IntoEnumIterator}; - - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let replica_count: u8 = 3; - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, - client_count: 1, - seed: 0xC0FF_EE00, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new( - usize::from(replica_count), - std::iter::once(client_id), - network_opts, - ); - let client = client::SimClient::new(client_id); - let ns_a = server_common::sharding::IggyNamespace::new(1, 1, 0); - let ns_b = server_common::sharding::IggyNamespace::new(1, 1, 1); - sim.init_partition(ns_a); - sim.init_partition(ns_b); - sim.register_client_with_primary(&client); - - assert_eq!(Action::COUNT, 23, "Action::COUNT changed; adjust weights"); - let entries: Vec<(Action, u8)> = Action::iter() - .enumerate() - .map(|(index, action)| (action, if index < 8 { 5 } else { 4 })) - .collect(); - let mut options = WorkloadOptions::new(0xC0FF_EE00, replica_count, vec![ns_a, ns_b]); - options.weights = ActionWeights::new(&entries); - let mut wl = Workload::new(options); - - // `run` asserts the invariants every tick; any regression panics - // here, replayable from the seed above. - let clients = [client]; - let replies = workload::run(&mut sim, &mut wl, &clients, 2_000, u64::MAX); - assert!( - replies > 0, - "uniform-weight workload produced no replies; invariants never exercised" - ); - - // Non-vacuity: the checks must have read live state for every pair. - let mut probe = Invariants::new(); - probe.check(&sim, &wl); - assert_eq!( - probe.tracked_pairs(), - usize::from(replica_count) * 2, - "expected commit_offset tracked for every (replica, namespace) pair" - ); - } - - /// With a positive crash probability - /// the driver crashes followers (never the primary) but never below the - /// survivor floor, while the per-tick invariants stay green and the - /// surviving quorum keeps committing. - #[test] - fn crash_injection_spares_primary_and_keeps_quorum() { - use crate::workload::{ - self, Workload, - actions::Action, - options::{ActionWeights, WorkloadOptions}, - }; - - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let replica_count: u8 = 5; - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, - client_count: 1, - seed: 0xC0FF_EE00, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new( - usize::from(replica_count), - std::iter::once(client_id), - network_opts, - ); - let client = client::SimClient::new(client_id); - let ns_a = server_common::sharding::IggyNamespace::new(1, 1, 0); - sim.init_partition(ns_a); - sim.register_client_with_primary(&client); - - let mut options = WorkloadOptions::new(0xC0FF_EE00, replica_count, vec![ns_a]); - options.weights = ActionWeights::new(&[(Action::SendMessages, 100)]); - options.crash_per_tick_ratio = 0.05; - options.min_survivors = 3; // quorum of 5 - - let mut wl = Workload::new(options); - let clients = [client]; - // run() asserts the per-tick invariants every tick under injected crashes. - let replies = workload::run(&mut sim, &mut wl, &clients, 3_000, u64::MAX); - - let crashed = sim.crashed.len(); - assert!( - crashed >= 1, - "expected at least one crash injected over the run" - ); - assert!( - !sim.is_crashed(0), - "primary (replica 0) must never be crashed" - ); - assert!( - usize::from(replica_count) - crashed >= 3, - "must keep at least min_survivors=3 live (crashed={crashed})" - ); - assert!( - replies > 0, - "surviving quorum must keep committing under follower crashes" - ); - } - - /// After a mixed metadata + partition run drains, the shadow's - /// predicted streams equal the metadata committed on the leader (the - /// entity-oracle payoff of the name-keyed shadow). + /// After a mixed metadata and partition run drains, the shadow's predicted + /// streams equal the metadata committed on the leader. /// - /// Interleaves creates, deletes, and partition sends from a single client. - /// A send between two metadata ops once consumed a metadata request number - /// and gapped the next create into a permanent `RequestGap`; the `SimClient` - /// per-plane numbering fix keeps the metadata sequence contiguous, so the - /// mix now drains. The entity oracle still compares only against the leader: - /// a quorum-excluded backup has no idle catch-up yet, so full cross-replica - /// equality stays deferred (see [`oracle`]). + /// Interleaves creates, deletes and partition sends from one client. A send + /// between two metadata ops once consumed a metadata request number, gapping the + /// next create; `SimClient`'s per-plane numbering keeps the metadata sequence + /// contiguous, so the mix drains and the auditor never misattributes a partition + /// reply to a metadata entry. Compares + /// against the leader only: a quorum-excluded backup has no idle catch-up, so + /// full cross-replica equality stays deferred (see [`oracle`]). #[test] fn quiesce_stream_entity_oracle_matches_leader() { use crate::workload::{ @@ -1955,10 +1892,9 @@ mod tests { sim.register_client_with_primary(&client); let mut options = WorkloadOptions::new(0xC0FF_EE00, replica_count, vec![ns_a]); - // Interleave metadata creates/deletes with partition sends: the send - // between two metadata ops is the case that previously wedged the next - // create. Sends do not touch the metadata entity sets, so the entity - // oracle below still compares stream state cleanly. + // The send between two metadata ops is the case that previously wedged the + // next create. Sends do not touch the metadata entity sets, so the oracle + // below still compares stream state cleanly. options.weights = ActionWeights::new(&[ (Action::CreateStream, 50), (Action::DeleteStream, 25), @@ -1975,7 +1911,7 @@ mod tests { "system did not drain within the tick budget" ); // Cross-replica agreement + entity oracle (single client => strict). - oracle::assert_converged(&sim, &wl); + oracle::assert_converged(&sim, &mut wl); } /// Under follower crashes the surviving quorum still drains and @@ -2012,12 +1948,10 @@ mod tests { sim.init_partition(ns_a); sim.register_client_with_primary(&client); - // Partition-plane workload under crashes: SendMessages replies and the - // partition plane converges across survivors. Stays partition-only by - // design to isolate partition-offset convergence from metadata loss to a - // crashed primary; the metadata entity oracle runs in the no-crash test. - // Crash one follower (keep quorum slack: 5 replicas, floor 4, quorum 3), - // so commits still reach quorum and the run drains. + // Partition-only by design, isolating partition-offset convergence from + // metadata lost to a crashed primary; the entity oracle runs in the no-crash + // test. One follower crashed with quorum slack (5 replicas, floor 4, quorum + // 3), so commits still reach quorum and the run drains. let mut options = WorkloadOptions::new(0xC0FF_EE00, replica_count, vec![ns_a]); options.weights = ActionWeights::new(&[(Action::SendMessages, 100)]); options.crash_per_tick_ratio = 0.05; @@ -2036,24 +1970,25 @@ mod tests { oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000), "surviving quorum did not drain within the tick budget" ); - oracle::assert_converged(&sim, &wl); + oracle::assert_converged(&sim, &mut wl); } - /// Drive Create-heavy then Delete-heavy workload; assert shadow tracks - /// live streams: + /// A lossy network drains, because the client resends. + /// + /// Without [`workload::Workload::due_resends`] this wedges immediately and + /// permanently: one in-flight slot per client, nothing times out, so the first + /// dropped request or reply strands that slot for the run. At 5% loss a 3000-tick + /// run drained a handful of replies and stopped, and `drive_to_quiesce` could + /// never finish, waiting on a reply the network had discarded. /// - /// - At least one `CreateStream` commits. - /// - At least one `DeleteStream` commits, proving sample picked a live - /// name (without shadow tracking, sample would return `None`). - /// - Shadow stream count matches net `creates - deletes`. + /// Asserts the resend path ran rather than the seed getting lucky. #[test] - fn shadow_tracks_live_streams() { + fn packet_loss_resends_and_drains() { use crate::workload::{ - Workload, - actions::Action, + self, Workload, options::{ActionWeights, WorkloadOptions}, + oracle, }; - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), @@ -2062,14 +1997,16 @@ mod tests { let replica_count: u8 = 3; let client_id: u128 = 1; + let seed = 0x105_5A1A; let network_opts = packet::PacketSimulatorOptions { node_count: replica_count, client_count: 1, - seed: 0x5EED_0002, + seed, + packet_loss_probability: 0.05, ..packet::PacketSimulatorOptions::default() }; let mut sim = Simulator::new( - replica_count as usize, + usize::from(replica_count), std::iter::once(client_id), network_opts, ); @@ -2078,346 +2015,602 @@ mod tests { sim.init_partition(ns_a); sim.register_client_with_primary(&client); - // Phase 1: Create-heavy to populate the shadow. - let mut options = WorkloadOptions::new(0x5EED_0002, replica_count, vec![ns_a]); - options.weights = ActionWeights::new(&[(Action::CreateStream, 100)]); + let mut options = WorkloadOptions::new(seed, replica_count, vec![ns_a]); + options.weights = ActionWeights::partition_only(); let mut wl = Workload::new(options); - for _tick in 0..3_000u32 { - if let Some((target, msg)) = wl.build_request(&client) { - sim.submit_request(client.client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let cmds = wl.on_reply(&reply); - apply_sim_commands(&mut sim, &cmds); - } - } - let created = wl.auditor.stats().commits_per_action[Action::CreateStream as usize]; - assert!(created > 0, "Create-only workload produced no commits"); - assert_eq!( - wl.shadow.stream_names.len() as u64, - created, - "shadow stream count diverged from CreateStream commits" - ); - // Phase 2: Create/Delete mix. DeleteStream sample succeeds only if - // shadow.pick_stream_name returns Some (the wiring under test). - wl.options.weights = - ActionWeights::new(&[(Action::CreateStream, 30), (Action::DeleteStream, 70)]); - for _tick in 0..3_000u32 { - if let Some((target, msg)) = wl.build_request(&client) { - sim.submit_request(client.client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let cmds = wl.on_reply(&reply); - apply_sim_commands(&mut sim, &cmds); - } - } - let deleted = wl.auditor.stats().commits_per_action[Action::DeleteStream as usize]; - let created_total = wl.auditor.stats().commits_per_action[Action::CreateStream as usize]; + let clients = [client]; + let replies = workload::run(&mut sim, &mut wl, &clients, 3_000, u64::MAX); + assert!(replies > 0, "lossy workload produced no replies"); assert!( - deleted > 0, - "DeleteStream never committed; shadow-driven sampling is broken \ - (sample would return None unless pick_stream_name found a live name)" + wl.resends() > 0, + "no request timed out at 5% packet loss, so the resend path never ran; \ + raise the loss rate or lower request_timeout_ticks" ); - let expected_live = created_total.saturating_sub(deleted); - assert_eq!( - wl.shadow.stream_names.len() as u64, - expected_live, - "shadow.stream_names.len() ({}) != creates ({}) - deletes ({}) = {}", - wl.shadow.stream_names.len(), - created_total, - deleted, - expected_live, + + assert!( + oracle::drive_to_quiesce(&mut sim, &mut wl, 20_000), + "{}", + oracle::quiesce_failure_report(&sim, &wl), ); + oracle::assert_converged(&sim, &mut wl); } - /// Drive workload with 4 concurrent clients over two namespaces; assert: - /// - /// - Every client observes at least one commit (no starvation). - /// - Per-(client, namespace) commit-monotonic invariant holds. - /// - Commits interleave across clients. - #[test] - fn multi_client_interleaves_commits() { + fn workload_hash_for_seed(seed: u64) -> u64 { + workload_hash(seed, 1).0 + } + + /// Reply-trace and executor-schedule hashes for a full workload run at + /// `shards_per_replica` shards. Shared by the single-shard locked + /// baseline and the multi-shard replay tests. + fn workload_hash(seed: u64, shards_per_replica: u16) -> (u64, u64) { use crate::workload::{ Workload, actions::Action, options::{ActionWeights, WorkloadOptions}, }; - - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); + use std::hash::{DefaultHasher, Hash, Hasher}; let replica_count: u8 = 3; - let client_ids: Vec = (1..=4).collect(); + let client_id: u128 = 1; let network_opts = packet::PacketSimulatorOptions { node_count: replica_count, - client_count: u8::try_from(client_ids.len()).expect("fits"), - seed: 0x5EED_0005, + client_count: 1, + seed, ..packet::PacketSimulatorOptions::default() }; - let mut sim = Simulator::new( + let mut sim = Simulator::with_shards( replica_count as usize, - client_ids.iter().copied(), + shards_per_replica, + std::iter::once(client_id), network_opts, ); - let clients: Vec = client_ids - .iter() - .map(|&id| client::SimClient::new(id)) - .collect(); - let ns_a = server_common::sharding::IggyNamespace::new(1, 1, 0); - let ns_b = server_common::sharding::IggyNamespace::new(1, 1, 1); + let client = SimClient::new(client_id); + + let ns_a = IggyNamespace::new(1, 1, 0); + let ns_b = IggyNamespace::new(1, 1, 1); sim.init_partition(ns_a); sim.init_partition(ns_b); - for c in &clients { - sim.register_client_with_primary(c); - } + sim.register_client_with_primary(&client); - let mut options = WorkloadOptions::new(0x5EED_0005, replica_count, vec![ns_a, ns_b]); - options.client_count = u8::try_from(clients.len()).expect("fits"); + let mut options = WorkloadOptions::new(seed, replica_count, vec![ns_a, ns_b]); options.weights = ActionWeights::new(&[ (Action::CreateStream, 5), (Action::SendMessages, 70), (Action::StoreConsumerOffset, 25), ]); - let mut wl = Workload::new(options); - let mut commits_per_client: std::collections::HashMap = - std::collections::HashMap::new(); + let mut wl = Workload::new(options); + let mut hasher = DefaultHasher::new(); let mut replies_seen = 0u64; - for _tick in 0..4_000u32 { - for c in &clients { - if let Some((target, msg)) = wl.build_request(c) { - sim.submit_request(c.client_id(), target, msg.into_generic()); - } - } - for reply in sim.step() { - let client_id = reply.header().client; + + // Inline driver: hash each reply tuple, so divergence is caught at the first + // non-matching reply rather than in end-of-run aggregates. The cap sits inside + // the per-reply loop so a multi-reply tick at `replies_seen=49` cannot leak a + // 50th into the hash. + 'outer: for _tick in 0..5_000u32 { + if let Some((target, msg)) = wl.build_request(&client) { + sim.submit_request(client.client_id(), target, msg.into_generic()); + } + for reply in sim.step() { + let h = reply.header(); + (h.client, h.request, h.op, h.commit, h.operation as u8).hash(&mut hasher); let cmds = wl.on_reply(&reply); apply_sim_commands(&mut sim, &cmds); - *commits_per_client.entry(client_id).or_insert(0) += 1; replies_seen += 1; + if replies_seen >= 50 { + break 'outer; + } } } - assert!( replies_seen > 0, - "multi-client workload produced no replies" - ); - for &id in &client_ids { - let count = commits_per_client.get(&id).copied().unwrap_or(0); - assert!( - count > 0, - "client {id} observed no commits; multi-client routing is starving \ - (counts: {commits_per_client:?})", - ); - } - let distinct = commits_per_client.values().filter(|&&c| c > 0).count(); - assert!( - distinct >= 2, - "commits concentrated on a single client ({commits_per_client:?}); \ - no interleaving observed" + "workload produced no replies; driver / sim wiring is broken" ); + + replies_seen.hash(&mut hasher); + wl.shadow.sends_committed(ns_a).hash(&mut hasher); + wl.shadow.sends_committed(ns_b).hash(&mut hasher); + // Catches PRNG-trace shifts from `sample` returning `None`. + // Stays 0 on the current seed mix; non-zero drifts the baseline. + wl.samples_none().hash(&mut hasher); + (hasher.finish(), sim.schedule_hash()) } - /// Outcome-first generation: with a single client the strict equality oracle - /// is on, so any targeted-vs-committed mismatch panics in `on_reply`. Populate - /// streams, then drive a Create/Delete mix targeting error outcomes - /// (`NameAlreadyExists` by reusing a live name, `StreamNotFound` by fabricating - /// an absent one). Assert the server committed rejections, proving the error - /// paths are generated and verified end-to-end. - #[test] - fn outcome_first_generation_commits_targeted_rejections() { - use crate::workload::{ - Workload, - actions::Action, - options::{ActionWeights, WorkloadOptions}, - }; + /// No shard of any replica dropped an inter-shard frame. Runs without injected + /// loss must keep the counters at zero; non-zero means an inbox silently shed a + /// frame, from undersized capacity or a routing bug, which would otherwise hide + /// behind VSR retransmit. + /// + /// `park_overflow` is deliberately NOT excluded. The reconciler is unwired here + /// (`init_partition` mirrors its outcome directly), so nothing drains the park + /// buffer and a parked frame is never re-dispatched or swept. Non-zero + /// `park_overflow` therefore means frames shed for a namespace that will never + /// materialise, which is the fault class this catches rather than the + /// back-pressure it would be in production. + /// + /// Same reason the park buffer must be empty at quiescence: a frame still parked + /// has no drainer, so it is neither delivered nor answered. + fn assert_no_frame_drops(sim: &Simulator) { + for (replica_idx, replica) in sim.replicas.iter().enumerate() { + for (shard_idx, shard) in replica.shards.iter().enumerate() { + assert_eq!( + shard.metrics().frame_drops_value(), + 0, + "replica {replica_idx} shard {shard_idx} dropped frames without injected loss" + ); + assert!( + shard.parked_namespaces().is_empty(), + "replica {replica_idx} shard {shard_idx} left partition frames parked; the \ + simulator wires no reconciler, so nothing will deliver or answer them" + ); + } + } + } + /// Drive a full dispatch-shell round-trip for `seed`: seed a partition + /// plus its metadata, log a client in against root, produce one message, + /// then poll. Returns the poll reply's raw bytes and the schedule hash. + fn shell_produce_poll(seed: u64) -> (Vec, u64) { server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), bucket_capacity: 1, }); - let replica_count: u8 = 3; let client_id: u128 = 1; let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, + node_count: 3, client_count: 1, - seed: 0x5EED_0E4C, + seed, ..packet::PacketSimulatorOptions::default() }; - let mut sim = Simulator::new( - replica_count as usize, - std::iter::once(client_id), - network_opts, - ); - let client = client::SimClient::new(client_id); - let ns_a = server_common::sharding::IggyNamespace::new(1, 1, 0); - sim.init_partition(ns_a); - sim.register_client_with_primary(&client); + let mut sim = Simulator::with_shards_shell(3, 1, std::iter::once(client_id), network_opts); + let ns = IggyNamespace::new(0, 0, 0); + sim.init_partition(ns); + sim.seed_stream_topic_partition(ns); - // Phase 1: populate streams so `NameAlreadyExists` has live targets. - let mut options = WorkloadOptions::new(0x5EED_0E4C, replica_count, vec![ns_a]); - options.weights = ActionWeights::new(&[(Action::CreateStream, 100)]); - let mut wl = Workload::new(options); - for _tick in 0..2_000u32 { - if let Some((target, msg)) = wl.build_request(&client) { - sim.submit_request(client.client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let cmds = wl.on_reply(&reply); - apply_sim_commands(&mut sim, &cmds); - } + let client = SimClient::new(client_id); + sim.shell_login(&client); + + // Produce through the shell too: `SimClient` emits the legacy + // `SendMessagesHeader` shape the real SDK sends, so + // `resolve_partition_request_namespace` decodes it on the + // `handle_client_request` path. Write and poll both hit real dispatch. + let payload = Bytes::from_static(b"shell-poll-payload"); + let produce = client.send_messages(ns, std::slice::from_ref(&payload)); + sim.submit_request(client_id, 0, produce.into_generic()); + for _ in 0..200 { + sim.step(); } - // Phase 2: keep creating (now hitting `NameAlreadyExists` on live names) - // and deleting (hitting `StreamNotFound` on fabricated names). - wl.options.weights = - ActionWeights::new(&[(Action::CreateStream, 50), (Action::DeleteStream, 50)]); - for _tick in 0..4_000u32 { - if let Some((target, msg)) = wl.build_request(&client) { - sim.submit_request(client.client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let cmds = wl.on_reply(&reply); - apply_sim_commands(&mut sim, &cmds); + // Poll through the dispatch shell (`on_client_request`, drain, + // `handle_poll_messages`, `partition_read`, `on_partition_read`), running as a + // task the executor interleaves with the pump. + let poll = client.poll_messages(ns, 10); + sim.submit_request(client_id, 0, poll.into_generic()); + let mut poll_reply = None; + for _ in 0..200 { + if let Some(reply) = sim.step().into_iter().next() { + poll_reply = Some(reply); + break; } } + let poll_reply = poll_reply.expect("shell poll: no reply within 200 steps"); + (poll_reply.as_slice().to_vec(), sim.schedule_hash()) + } + /// A `SimClient` poll returns the produced messages through the real dispatch + /// read path (`on_client_request`, `handle_poll_messages`, `partition_read`, + /// `on_partition_read`), running as a task the executor interleaves with the pump, + /// and the whole login/produce/poll round-trip replays byte-for-byte on one seed. + #[test] + fn shell_poll_returns_produced_messages_deterministically() { + const PAYLOAD: &[u8] = b"shell-poll-payload"; + let (reply_a, schedule_a) = shell_produce_poll(0x5CED_0011); + let (reply_b, schedule_b) = shell_produce_poll(0x5CED_0011); assert!( - wl.auditor.stats().committed_rejections > 0, - "outcome-first generation produced no committed rejections; error \ - outcomes are not being targeted, or the server is not rejecting them" + reply_a + .windows(PAYLOAD.len()) + .any(|window| window == PAYLOAD), + "poll reply did not carry the produced payload through the real read handler" + ); + assert!( + reply_b + .windows(PAYLOAD.len()) + .any(|window| window == PAYLOAD), + "second run's poll reply did not carry the produced payload" + ); + // The produce stamps a random UUID per message (`random_id::get_uuid`), so + // reply bytes differ run to run while the seeded executor schedule must + // replay. Same reason the workload tests hash reply headers, not bodies. + assert_eq!( + schedule_a, schedule_b, + "shell schedule diverged at same seed" ); } - fn workload_hash_for_seed(seed: u64) -> u64 { - workload_hash(seed, 1).0 - } + /// The dispatch shell's reason to exist: detect the PR #3557 async-concurrency + /// class, a partition reference held across an `.await` while a sibling task + /// mutates the partitions vec. Under the deterministic executor a parked read with + /// a live borrow IS a borrow held across a suspension, so a concurrent `remove` + /// trips the `#[cfg(debug_assertions)]` tripwire. The correct `with_partition` + /// read drops the borrow first, so the same interleaving is sound. Debug-only, as + /// is the `BorrowGuard` it rides on. + /// + /// Injected through the synthetic `hold_borrow_across_await` rather than the real + /// read, because the production read has no borrow-holding suspension to seed: the + /// journal read is a synchronous memory copy, and `with_partition` returns an owned + /// `PollPlan` before the only awaits (disk read, offset persist) run off the borrow + /// in `spawn_poll_io`. + /// + /// TODO: once storage faults are modelled, the disk-tier read + /// (`PollPlan::execute`, `read_disk`) becomes a real seedable await in the read + /// path. A regression holding a borrow across it, against a concurrent reconcile + /// `InsertOwned` reallocation, would trip this through the real handler and retire + /// the synthetic seam. + #[cfg(debug_assertions)] + #[test] + fn shell_detects_partition_borrow_held_across_await() { + use crate::executor::DetExecutor; + use consensus::PartitionsHandle; + use std::panic::{AssertUnwindSafe, catch_unwind}; - /// Reply-trace and executor-schedule hashes for a full workload run at - /// `shards_per_replica` shards. Shared by the single-shard locked - /// baseline and the multi-shard replay tests. - fn workload_hash(seed: u64, shards_per_replica: u16) -> (u64, u64) { - use crate::workload::{ - Workload, - actions::Action, - options::{ActionWeights, WorkloadOptions}, - }; - use std::hash::{DefaultHasher, Hash, Hasher}; + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); - let replica_count: u8 = 3; - let client_id: u128 = 1; let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, + node_count: 3, client_count: 1, - seed, + seed: 0x5CED_0021, ..packet::PacketSimulatorOptions::default() }; + let mut sim = Simulator::new(3, std::iter::once(1u128), network_opts); + let ns_a = IggyNamespace::new(0, 0, 0); + let ns_b = IggyNamespace::new(0, 0, 1); + sim.init_partition(ns_a); + sim.init_partition(ns_b); - let mut sim = Simulator::with_shards( - replica_count as usize, - shards_per_replica, - std::iter::once(client_id), - network_opts, + // BAD read: holds a partition borrow across a suspension. The mutator runs + // while it is parked with the borrow live, so the tripwire fires. + // `catch_unwind` builds the executor inline, so unwinding drops the parked + // read's guard and restores the borrow count for the next case. + let tripped = catch_unwind(AssertUnwindSafe(|| { + let mut executor = DetExecutor::new(7); + let read = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + read.plane + .partitions() + .hold_borrow_across_await(std::future::pending()) + .await; + }); + executor.run_until_stalled(POLL_BUDGET); // borrow acquired; task parks + let mutate = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + mutate.plane.partitions().remove(&ns_b); + }); + executor.run_until_stalled(POLL_BUDGET); // mutate while borrow live + })) + .is_err(); + assert!( + tripped, + "borrow-held-across-await went undetected: the concurrent mutation \ + did not trip the #3557 borrow tripwire under the executor" + ); + // The tripwire fires BEFORE `remove` touches the vec, its assert being the + // first statement, so the detector aborts the mutation that would have + // dangled the live borrow. Both partitions survive intact: the class is caught + // before it can corrupt state. + assert!( + sim.offsets(0, ns_a).is_some() && sim.offsets(0, ns_b).is_some(), + "tripwire must abort the mutation before it corrupts the partitions vec" ); - let client = SimClient::new(client_id); - let ns_a = IggyNamespace::new(1, 1, 0); - let ns_b = IggyNamespace::new(1, 1, 1); + // REAL read: `with_partition` scopes the borrow, dropping it before the + // suspension, so the identical interleaving is sound (no tripwire). + let mut executor = DetExecutor::new(7); + let read = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + let _ = read + .plane + .partitions() + .with_partition(&ns_a, |_partition| ()); + std::future::pending::<()>().await; + }); + executor.run_until_stalled(POLL_BUDGET); + let mutate = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + mutate.plane.partitions().remove(&ns_b); + }); + executor.run_until_stalled(POLL_BUDGET); + // The mutation the bad read's tripwire aborted now applies cleanly, no borrow + // being live across the suspension: `ns_b` gone, `ns_a` intact, so the correct + // read is sound under the same schedule. + assert!( + sim.offsets(0, ns_a).is_some() && sim.offsets(0, ns_b).is_none(), + "correct with_partition read must leave the concurrent remove sound" + ); + } + + /// The realloc half of the PR #3557 class, and the stronger one: a pump `insert` + /// that grows the partitions vec MOVES every element, so a stale reference to ANY + /// partition dangles, not just the one a `swap_remove` displaced. + /// `shell_detects_partition_borrow_held_across_await` covers the remove; this + /// covers the grow, on the same two-task interleave (reader parked with a live + /// borrow, pump-shaped task mutating the container under it). + /// + /// The buffer address is asserted to have MOVED, so this cannot pass on a push + /// into spare capacity, which relocates nothing. Debug-only, like the + /// `BorrowGuard` it drives. + #[cfg(debug_assertions)] + #[test] + fn shell_detects_partition_borrow_held_across_a_pump_realloc() { + use crate::executor::DetExecutor; + use consensus::PartitionsHandle; + use std::panic::{AssertUnwindSafe, catch_unwind}; + + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + + let network_opts = packet::PacketSimulatorOptions { + node_count: 3, + client_count: 1, + seed: 0x5CED_0022, + ..packet::PacketSimulatorOptions::default() + }; + let mut sim = Simulator::new(3, std::iter::once(1u128), network_opts); + let ns_a = IggyNamespace::new(0, 0, 0); + let ns_b = IggyNamespace::new(0, 0, 1); + // The namespace the pump grows the vec with. Never materialised up front: + // inserting it IS the mutation under test. + let ns_grow = IggyNamespace::new(0, 0, 2); sim.init_partition(ns_a); sim.init_partition(ns_b); - sim.register_client_with_primary(&client); - let mut options = WorkloadOptions::new(seed, replica_count, vec![ns_a, ns_b]); - options.weights = ActionWeights::new(&[ - (Action::CreateStream, 5), - (Action::SendMessages, 70), - (Action::StoreConsumerOffset, 25), - ]); + // BAD read: the borrow is live across the suspension, so the pump's growing + // insert lands while a stale reference to every partition is outstanding. + // `catch_unwind` builds the executor inline, so unwinding drops the parked + // read's guard and restores the borrow count. + let tripped = catch_unwind(AssertUnwindSafe(|| { + let mut executor = DetExecutor::new(11); + let read = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + read.plane + .partitions() + .hold_borrow_across_await(std::future::pending()) + .await; + }); + executor.run_until_stalled(POLL_BUDGET); // borrow acquired; task parks + let grow = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + grow.init_partition(ns_grow, None, None, None, false); + }); + executor.run_until_stalled(POLL_BUDGET); // grow while the borrow is live + })) + .is_err(); + assert!( + tripped, + "a pump realloc under a live partition borrow went undetected: the \ + #3557 tripwire did not fire on the growing insert" + ); + // The tripwire asserts before `push`, so the vec is untouched: the two + // originals survive and the grow namespace never materialised. + let partitions = sim.replicas[0].shards[0].plane.partitions(); + assert_eq!( + partitions.len(), + 2, + "tripwire must abort the insert before it relocates the vec" + ); + assert!(!partitions.contains(&ns_grow)); - let mut wl = Workload::new(options); - let mut hasher = DefaultHasher::new(); - let mut replies_seen = 0u64; + // REAL read: `with_partition` drops the borrow before the suspension, so + // the identical schedule is sound and the grow applies. + let addr_before = partitions.buffer_addr(); + let mut executor = DetExecutor::new(11); + let read = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + let _ = read + .plane + .partitions() + .with_partition(&ns_a, |_partition| ()); + std::future::pending::<()>().await; + }); + executor.run_until_stalled(POLL_BUDGET); + let grow = Rc::clone(&sim.replicas[0].shards[0]); + executor.spawn(async move { + grow.init_partition(ns_grow, None, None, None, false); + }); + executor.run_until_stalled(POLL_BUDGET); - // Inline driver: hash each reply tuple so divergence is caught at - // the first non-matching reply, not just end-of-run aggregates. - // The reply cap lives inside the per-reply loop so a multi-reply - // tick at replies_seen=49 cannot leak a 50th+ reply into the hash. - 'outer: for _tick in 0..5_000u32 { - if let Some((target, msg)) = wl.build_request(&client) { - sim.submit_request(client.client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let h = reply.header(); - (h.client, h.request, h.op, h.commit, h.operation as u8).hash(&mut hasher); - let cmds = wl.on_reply(&reply); - apply_sim_commands(&mut sim, &cmds); - replies_seen += 1; - if replies_seen >= 50 { - break 'outer; - } + let partitions = sim.replicas[0].shards[0].plane.partitions(); + assert!( + partitions.contains(&ns_grow), + "correct with_partition read must leave the concurrent insert sound" + ); + assert_ne!( + partitions.buffer_addr(), + addr_before, + "the insert landed in spare capacity, so nothing moved and this test \ + proves nothing about a realloc; seed more partitions before the grow" + ); + // Every pre-existing partition is still addressable after the move, which + // is what a stale reference would have missed. + assert!(partitions.contains(&ns_a) && partitions.contains(&ns_b)); + } + + /// Committed stream names on one replica, read out of the committed (left) + /// buffer so an uncommitted write is invisible. + fn committed_stream_names( + sim: &Simulator, + replica_idx: usize, + ) -> std::collections::BTreeSet { + use metadata::impls::metadata::StreamsFrontend; + sim.replicas[replica_idx].shards[0] + .plane + .metadata() + .mux_stm + .streams() + .read(|inner| { + inner + .items + .iter() + .map(|(_, stream)| stream.name.to_string()) + .collect() + }) + } + + /// How much of the WAL a checkpoint reclaimed. Non-zero is the precondition every + /// checkpoint-recovery test needs: at zero the WAL still holds everything. + fn snapshot_floor(sim: &Simulator, replica_idx: usize) -> u64 { + use journal::Journal; + sim.replicas[replica_idx].metadata_journal.snapshot_op() + } + + /// A client's committed request watermark on one replica. + fn client_watermark(sim: &Simulator, replica_idx: usize, client_id: u128) -> Option { + sim.replicas[replica_idx].shards[0] + .plane + .metadata() + .client_table + .borrow() + .get_watermark(client_id) + } + + /// A checkpointing cluster with `streams` committed streams behind it, named + /// `wl-{prefix}-N`. The returned `TempDir` keeps the snapshots alive. + fn checkpointing_cluster( + replicas: u8, + seed: u64, + prefix: &str, + streams: u32, + ) -> (Simulator, u128, tempfile::TempDir) { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + let root = tempfile::tempdir().expect("temp dir for the simulator's snapshots"); + let client_id: u128 = 1; + let mut sim = Simulator::with_checkpoints( + usize::from(replicas), + std::iter::once(client_id), + packet::PacketSimulatorOptions { + node_count: replicas, + client_count: 1, + seed, + ..packet::PacketSimulatorOptions::default() + }, + false, + root.path(), + ); + // Small enough that the ops below cross the coordinator's margin. + sim.set_metadata_journal_slots(80); + + let client = SimClient::new(client_id); + sim.register_client_with_primary(&client); + for sequence in 0..streams { + let msg = client.create_stream(&format!("wl-{prefix}-{sequence}")); + sim.submit_request(client_id, 0, msg.into_generic()); + for _ in 0..40 { + sim.step(); } } + (sim, client_id, root) + } + + /// A solo replica that checkpointed recovers the state the checkpoint absorbed. + /// + /// The case with no second opinion: a clustered replica repairs a botched local + /// recovery from a peer, so what boot reconstructs here IS the state. The client + /// table is asserted too, being folded in separately by `persist_snapshot`. + #[test] + fn solo_replica_recovers_the_state_its_checkpoint_absorbed() { + let (mut sim, client_id, _root) = checkpointing_cluster(1, 0xC4E0_0002, "solo", 40); + + let before = committed_stream_names(&sim, 0); + let watermark_before = client_watermark(&sim, 0, client_id); assert!( - replies_seen > 0, - "workload produced no replies; driver / sim wiring is broken" + snapshot_floor(&sim, 0) > 0, + "the solo replica never checkpointed, so this proves nothing about \ + snapshot recovery" ); - replies_seen.hash(&mut hasher); - wl.shadow.sends_committed(ns_a).hash(&mut hasher); - wl.shadow.sends_committed(ns_b).hash(&mut hasher); - // Catches PRNG-trace shifts from `sample` returning `None`. - // Stays 0 on the current seed mix; non-zero drifts the baseline. - wl.samples_none().hash(&mut hasher); - (hasher.finish(), sim.schedule_hash()) + sim.replica_crash(0); + sim.replica_restart(0); + for _ in 0..2_000 { + sim.step(); + } + + assert_eq!( + committed_stream_names(&sim, 0), + before, + "the restarted solo replica lost committed streams the checkpoint \ + drained out of the WAL" + ); + assert_eq!( + client_watermark(&sim, 0, client_id), + watermark_before, + "the checkpoint's folded client table did not come back, so a session \ + below the snapshot floor lost its watermark" + ); } - /// Assert no shard of any replica dropped an inter-shard frame. Runs - /// without injected loss must keep the counters at zero; a non-zero - /// value means an inbox silently shed a frame (undersized capacity or - /// a routing bug), which would otherwise hide behind VSR retransmit. + /// A clustered replica that took its own checkpoint rejoins holding the same + /// committed metadata as a healthy peer. /// - /// `park_overflow` is deliberately NOT excluded. The simulator never wires - /// the partition reconciler (`init_partition` mirrors its outcome directly), - /// so nothing here ever drains the park buffer: a parked frame is never - /// re-dispatched and never swept. A non-zero `park_overflow` in the simulator - /// therefore means frames were shed for a namespace that will never - /// materialise -- which is the very fault class this assert exists to catch, - /// not the back-pressure it would be in production. - /// - /// For the same reason the park buffer must be empty at quiescence: a frame - /// still parked here has no drainer, so it will neither be delivered nor - /// answered. - fn assert_no_frame_drops(sim: &Simulator) { - for (replica_idx, replica) in sim.replicas.iter().enumerate() { - for (shard_idx, shard) in replica.shards.iter().enumerate() { - assert_eq!( - shard.metrics().frame_drops_value(), - 0, - "replica {replica_idx} shard {shard_idx} dropped frames without injected loss" - ); - assert!( - shard.parked_namespaces().is_empty(), - "replica {replica_idx} shard {shard_idx} left partition frames parked; the \ - simulator wires no reconciler, so nothing will deliver or answer them" - ); - } + /// Against a peer, not a recorded snapshot of itself: a replica that dropped the + /// drained prefix still reports a plausible commit point, and only the peer + /// comparison shows the state behind it is wrong. + #[test] + fn a_checkpointed_replica_rejoins_agreeing_with_a_healthy_peer() { + let (mut sim, _client_id, _root) = checkpointing_cluster(3, 0xC4E0_0003, "peer", 40); + + // A backup, so the restart does not also trigger a view change: the subject + // here is local recovery, not election. + let rejoining = 1u8; + assert!( + snapshot_floor(&sim, usize::from(rejoining)) > 0, + "replica {rejoining} never checkpointed, so its restart exercises no \ + snapshot recovery" + ); + let healthy = committed_stream_names(&sim, 0); + assert!( + healthy.len() > 1, + "the peer holds no workload streams to compare" + ); + + sim.replica_crash(rejoining); + sim.replica_restart(rejoining); + for _ in 0..8_000 { + sim.step(); } + + assert_eq!( + committed_stream_names(&sim, usize::from(rejoining)), + healthy, + "the rejoined replica disagrees with a healthy peer on committed \ + metadata: local recovery dropped the prefix its checkpoint drained" + ); } - /// A partition materialises only on its murmur3 owner shard; every - /// shard of a replica carries an identical routing row; metadata - /// consensus/journal state exists only on shard 0. + /// A namespace still live in committed metadata but hosted by nobody is a + /// convergence FAILURE, not a converged cluster. + /// + /// Settlement and the leader-relative offset check both skip such a namespace, + /// which is right for a deleted stream and used to be the only word on the + /// subject. Seeds the metadata half without the partition half, the state a total + /// loss of instances leaves. #[test] - fn multi_shard_partition_and_metadata_placement() { - use consensus::MetadataHandle; - use shard::shards_table::{ShardsTable, calculate_shard_assignment}; + #[should_panic(expected = "no live replica hosts it at quiesce")] + fn a_live_namespace_with_no_host_fails_convergence() { + use crate::workload::{Workload, options::WorkloadOptions, oracle}; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, @@ -2425,50 +2618,46 @@ mod tests { bucket_capacity: 1, }); - let shards_per_replica: u16 = 4; - let network_opts = packet::PacketSimulatorOptions { - node_count: 3, - client_count: 1, - seed: 0x5EED_5AAD, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = - Simulator::with_shards(3, shards_per_replica, std::iter::once(1), network_opts); - let ns = IggyNamespace::new(1, 1, 0); - sim.init_partition(ns); - - let owner = calculate_shard_assignment(&ns, u32::from(shards_per_replica)); - for replica in &sim.replicas { - for (shard_idx, shard) in replica.shards.iter().enumerate() { - assert_eq!( - shard.plane.partitions().contains(&ns), - shard_idx == usize::from(owner), - "partition must live exactly on its hash owner (owner={owner})" - ); - assert_eq!( - shard.shards_table().shard_for(ns), - Some(owner), - "every shard must carry the same routing row" - ); - assert_eq!( - shard.plane.metadata().consensus.is_some(), - shard_idx == 0, - "metadata consensus must exist only on shard 0" - ); - } + let replica_count: u8 = 3; + let client_id: u128 = 1; + let seed = 0xC4E0_0004; + let mut sim = Simulator::new( + usize::from(replica_count), + std::iter::once(client_id), + packet::PacketSimulatorOptions { + node_count: replica_count, + client_count: 1, + seed, + ..packet::PacketSimulatorOptions::default() + }, + ); + let client = SimClient::new(client_id); + let ns = server_common::sharding::IggyNamespace::new(1, 1, 0); + // Metadata only: the namespace is committed-visible, but no replica ever + // materialises the group (`init_partition` is deliberately not called). + sim.seed_stream_topic_partition(ns); + sim.register_client_with_primary(&client); + for _ in 0..200 { + sim.step(); } + + let mut workload = Workload::new(WorkloadOptions::new(seed, replica_count, vec![ns])); + oracle::assert_converged(&sim, &mut workload); } - /// Single-writer metadata: a peer shard resolves a namespace against shard - /// 0's metadata through the shared left-right read handle. Before this, - /// every shard carried an independent writable STM, so a write that reached - /// only shard 0 (as a metadata consensus commit does) was invisible to - /// peers and a partition op homing on a peer shard failed to resolve its - /// namespace. Seeding only shard 0 and reading it back from every peer is - /// the direct proof of the read-handle propagation. + /// A replica that checkpoints serves a real state transfer: the rejoining peer + /// fetches the snapshot in chunks rather than stalling at the handshake. + /// + /// Companion to + /// [`repair_below_the_snapshot_floor_escalates_to_state_transfer`], which stamps a + /// watermark without snapshot bytes and so reaches only `StateTransferTarget`. + /// Here the coordinator is armed with a data directory and the journal is bounded, + /// so the cluster checkpoints for real. Both were needed and neither was present: + /// no directory meant no `SnapshotCoordinator`, and an unbounded journal never + /// fired `should_checkpoint`. #[test] - fn peer_shard_resolves_namespace_via_shard0_read_handle() { - use iggy_binary_protocol::WireIdentifier; + fn checkpointing_cluster_serves_a_chunked_state_transfer() { + use iggy_binary_protocol::Command; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, @@ -2476,413 +2665,485 @@ mod tests { bucket_capacity: 1, }); + let root = tempfile::tempdir().expect("temp dir for the simulator's snapshots"); + let replica_count: u8 = 3; + let client_id: u128 = 1; let network_opts = packet::PacketSimulatorOptions { - node_count: 1, + node_count: replica_count, client_count: 1, - seed: 0x5EED_B00C, + seed: 0xC4E0_0001, ..packet::PacketSimulatorOptions::default() }; - // One replica, four shards: shard 0 writes metadata, shards 1..4 read it. - let sim = Simulator::with_shards(1, 4, std::iter::once(1), network_opts); + let mut sim = Simulator::with_checkpoints( + usize::from(replica_count), + std::iter::once(client_id), + network_opts, + false, + root.path(), + ); + // Small enough that the ops below cross the margin; the coordinator forces + // a checkpoint once free slots fall to its margin (64 by default). + sim.set_metadata_journal_slots(80); - // Seeds only shard 0's writable STM (see `seed_stream_topic_partition`). - let ns = IggyNamespace::new(0, 0, 0); - sim.seed_stream_topic_partition(ns); + let client = SimClient::new(client_id); + sim.register_client_with_primary(&client); - let resolve = |shard: &Rc| { - shard - .plane - .metadata() - .mux_stm - .streams() - .namespace_from_partition( - &WireIdentifier::numeric(0), - &WireIdentifier::numeric(0), - 0, - ) - }; + let lagging = 2u8; + sim.replica_crash(lagging); - let shards = &sim.replicas[0].shards; - let writer_resolved = resolve(&shards[0]); - assert_eq!( - writer_resolved, - Some(ns), - "shard 0 (metadata writer) must resolve the seeded namespace" - ); - for (shard_idx, peer) in shards.iter().enumerate().skip(1) { - assert_eq!( - resolve(peer), - writer_resolved, - "peer shard {shard_idx} must resolve via shard 0's shared read handle, \ - not an independent STM" - ); + // Commit past the checkpoint margin while the lagging replica is down, so + // the survivors checkpoint and compact the prefix it is missing. + for sequence in 0..40u32 { + let msg = client.create_stream(&format!("wl-checkpoint-{sequence}")); + sim.submit_request(client_id, 0, msg.into_generic()); + for _ in 0..40 { + sim.step(); + } + } + + let snapshot = root + .path() + .join("replica-0") + .join(metadata::impls::METADATA_DIR) + .join(metadata::impls::SNAPSHOT_FILE_NAME); + assert!( + snapshot.exists(), + "the primary never checkpointed, so there is no snapshot to transfer: \ + raise the op count or lower the journal slot count" + ); + + sim.replica_restart(lagging); + for _ in 0..8_000 { + sim.step(); } + + assert!( + sim.network.delivered_any(Command::RequestStateChunk), + "the rejoining replica never asked for a chunk, so the transfer \ + stalled at the handshake exactly as it does without a checkpoint" + ); + assert!( + sim.network.delivered_any(Command::StateChunk), + "no chunk was served: the peer offered a transfer it could not fulfil" + ); + + // Traffic is not recovery: everything above passes when the chunks arrive and + // the install then fails. Each assertion below closes one of those ways. + let recovered = &sim.replicas[usize::from(lagging)].shards[0]; + let consensus = recovered + .plane + .metadata() + .consensus + .as_ref() + .expect("shard 0 owns metadata consensus"); + assert_eq!( + consensus.status(), + consensus::Status::Normal, + "the rejoining replica never returned to Normal" + ); + let transferred_floor = snapshot_floor(&sim, 0); + assert!( + transferred_floor > 0, + "the serving peer has no snapshot to transfer" + ); + assert!( + consensus.commit_min() >= transferred_floor, + "the rejoining replica commits through {} but was served a snapshot \ + covering {transferred_floor}: the install did not land", + consensus.commit_min(), + ); + assert!( + consensus.commit_max() >= consensus.recovery_barrier(), + "a recovery barrier at {} still gates the rejoining replica (commit_max {})", + consensus.recovery_barrier(), + consensus.commit_max(), + ); + + assert_eq!( + committed_stream_names(&sim, usize::from(lagging)), + committed_stream_names(&sim, 0), + "the transfer left the rejoining replica holding different committed \ + metadata than the peer that served it" + ); + assert_eq!( + client_watermark(&sim, usize::from(lagging), client_id), + client_watermark(&sim, 0, client_id), + "the transferred client table did not install: the session below the \ + snapshot floor came back without its watermark" + ); } - /// View change at 5 replicas x 3 shards: crash the primary replica, - /// survivors elect a new primary for the partition group, and a - /// post-change send commits through the mesh. + /// A client that dials a BACKUP still gets a working session, and the login + /// travels as a forwarded consensus proposal rather than a redirect. + /// + /// Register forwarding exists so a client need not find the primary itself: the + /// backup verifies the credentials locally, sends only the proposal on as + /// `ForwardRegister`, parks the login until `ForwardRegisterResult` returns, then + /// answers on the connection it owns. The subsystem landed with no deterministic + /// coverage and could have none while the harness only dialed the primary. + /// + /// Asserts the four frames were delivered rather than inferring them from a working + /// session: dialing a backup would also "work" under a silent redirect, which is + /// the design this replaced. #[test] - fn multi_shard_view_change_after_primary_crash() { + fn login_via_backup_forwards_the_register_to_the_primary() { + use iggy_binary_protocol::Command; + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), bucket_capacity: 1, }); - let replica_count: u8 = 5; + let replica_count: u8 = 3; let client_id: u128 = 1; let network_opts = packet::PacketSimulatorOptions { node_count: replica_count, client_count: 1, - seed: 0x5EED_5C0C, + seed: 0xF02D_0001, ..packet::PacketSimulatorOptions::default() }; - let mut sim = Simulator::with_shards( - replica_count as usize, - 3, + let mut sim = Simulator::with_shards_shell( + usize::from(replica_count), + 1, std::iter::once(client_id), network_opts, ); - let client = SimClient::new(client_id); let ns = IggyNamespace::new(1, 1, 0); sim.init_partition(ns); - sim.register_client_with_primary(&client); - - let msg = client.send_messages(ns, &[Bytes::from_static(b"before crash")]); - sim.submit_request(client_id, 0, msg.into_generic()); - let mut got_reply = false; - for _ in 0..200 { - if !sim.step().is_empty() { - got_reply = true; - break; - } - } - assert!(got_reply, "expected reply before crash"); + sim.seed_stream_topic_partition(ns); - sim.replica_crash(0); - for _ in 0..800 { - sim.step(); - } + // Replica 0 leads both planes at view 0, so replica 1 is a backup and the + // login has to be forwarded. + let client = SimClient::new(client_id); + sim.shell_login_via(&client, 1); - let mut new_primary_found = false; - for replica_idx in 1..replica_count { - let consensus = sim.replicas[replica_idx as usize] - .partition_shard(ns) - .plane - .partitions() - .get_by_ns(&ns) - .expect("partition must exist on every live replica's owner shard") - .consensus(); - if consensus.view() > 0 - && consensus.status() == Status::Normal - && consensus.is_primary() + assert!( + sim.network.delivered_any(Command::ForwardRegister), + "no ForwardRegister crossed the wire: the backup answered the login \ + itself, so this covers nothing" + ); + assert!( + sim.network.delivered_any(Command::ForwardRegisterResult), + "the forwarded register was never answered, so the login below \ + succeeded by some other route" + ); + + // Log out on the SAME backup: covers the other half of forwarding and proves + // the session was real, only a bound session being torn down, with the + // teardown replicating through the primary as the register did. A logout, not + // a data request, because the session belongs to the connection and a backup + // refuses a partition write for routing reasons (`TransientNotAccepted`) that + // say nothing about the session. + let logout = client.logout(); + let request = logout.header().request; + sim.submit_request(client_id, 1, logout.into_generic()); + let mut answered = false; + for _ in 0..400 { + if let Some(reply) = sim + .step() + .into_iter() + .find(|reply| reply.header().request == request) { - new_primary_found = true; - } - } - assert!(new_primary_found, "expected a new primary after crash"); - - let live = sim.replicas[1].partition_shard(ns); - let live_consensus = live - .plane - .partitions() - .get_by_ns(&ns) - .expect("partition must exist on replica 1") - .consensus(); - let new_primary_idx = live_consensus.primary_index(live_consensus.view()); - let msg2 = client.send_messages(ns, &[Bytes::from_static(b"after view change")]); - sim.submit_request(client_id, new_primary_idx, msg2.into_generic()); - let mut got_reply_after = false; - for _ in 0..200 { - if !sim.step().is_empty() { - got_reply_after = true; + assert_eq!( + reply.header().status, + 0, + "logout on the forwarded session was refused (status {})", + reply.header().status, + ); + answered = true; break; } } - assert!(got_reply_after, "expected reply from new primary"); - } - - /// Multi-shard replay: the same seed reproduces both the reply trace - /// and the executor schedule; a different seed diverges in both. - #[test] - fn multi_shard_replay_is_deterministic() { - let (replies_a, schedule_a) = workload_hash(0xD0D0_0001, 4); - let (replies_b, schedule_b) = workload_hash(0xD0D0_0001, 4); - assert_eq!(replies_a, replies_b, "reply trace diverged at same seed"); - assert_eq!(schedule_a, schedule_b, "schedule diverged at same seed"); - - let (replies_c, schedule_c) = workload_hash(0xD0D0_0002, 4); - assert_ne!(replies_a, replies_c, "different seeds, identical replies"); - assert_ne!( - schedule_a, schedule_c, - "different seeds, identical schedule" + assert!(answered, "no reply to the logout issued on the backup"); + assert!( + sim.network.delivered_any(Command::ForwardLogout), + "the backup committed the logout without asking the primary" + ); + assert!( + sim.network.delivered_any(Command::ForwardLogoutResult), + "the forwarded logout was never answered" ); } - /// Schedule hash for `seed` after stepping the consensus plane with no - /// client traffic, with the dispatch shell on or off. - fn consensus_schedule_hash(seed: u64, shell: bool) -> u64 { + /// The workload drains and converges when every request goes through the + /// server's real dispatch handlers rather than the raw `on_message` path. + /// + /// Its own test because the shell path is where authorization, session binding and + /// the pre-commit deny replies live; the raw path has no deny site. Running the + /// workload here is the only thing that exercises them, and it surfaced that the + /// workload had never modelled a denial: nonzero `ReplyHeader::status` means an + /// empty body, and the decoder read a result section off it and called the reply + /// corrupt. + /// + /// Asserts denials were observed, so this cannot pass on a path where nothing is + /// denied. + #[test] + fn shell_workload_drains_and_converges() { + use crate::workload::{ + self, Workload, + options::{ActionWeights, WorkloadOptions}, + oracle, + }; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), bucket_capacity: 1, }); + let replica_count: u8 = 3; + let client_id: u128 = 1; + let seed = 0x5E11_0001; let network_opts = packet::PacketSimulatorOptions { - node_count: 3, + node_count: replica_count, client_count: 1, seed, ..packet::PacketSimulatorOptions::default() }; - let mut sim = if shell { - Simulator::with_shards_shell(3, 1, std::iter::once(1u128), network_opts) - } else { - Simulator::with_shards(3, 1, std::iter::once(1u128), network_opts) - }; - for _ in 0..20 { - sim.step(); - } - sim.schedule_hash() - } + let mut sim = Simulator::with_shards_shell( + usize::from(replica_count), + 1, + std::iter::once(client_id), + network_opts, + ); + let ns = IggyNamespace::new(1, 1, 0); + sim.init_partition(ns); + // The dispatch path resolves a partition request's namespace against + // committed metadata, so the stream and topic have to exist too. + sim.seed_stream_topic_partition(ns); - /// Turning the dispatch shell on wires the server's real deferred - /// handlers on every shard. With no client traffic none of them is - /// reached, so the consensus plane both replays deterministically and - /// matches the shell-off schedule: the toggle is genuinely off the - /// consensus path. Also guards that shell construction does not panic. - #[test] - fn shell_on_consensus_schedule_matches_shell_off() { - let seed = 0x5CED_0001; - assert_eq!( - consensus_schedule_hash(seed, true), - consensus_schedule_hash(seed, true), - "shell-on schedule diverged at same seed" + let client = SimClient::new(client_id); + // Log in rather than bare-register: dispatch admits a request only from a + // bound session. + sim.shell_login(&client); + + let mut options = WorkloadOptions::new(seed, replica_count, vec![ns]); + options.weights = ActionWeights::uniform(); + let mut wl = Workload::new(options); + + let clients = [client]; + let replies = workload::run(&mut sim, &mut wl, &clients, 4_000, u64::MAX); + assert!(replies > 0, "shell workload produced no replies"); + + let stats = wl.auditor.stats(); + assert!( + stats.commits_per_action.iter().sum::() > 0, + "shell workload committed nothing, so the dispatch path never got past \ + admission" ); - assert_eq!( - consensus_schedule_hash(seed, true), - consensus_schedule_hash(seed, false), - "shell perturbed the consensus schedule despite no client traffic" + assert!( + stats.denials > 0, + "no request was denied, so the pre-commit deny path this test exists to \ + cover never ran" ); - assert_ne!( - consensus_schedule_hash(0x5CED_0001, true), - consensus_schedule_hash(0x5CED_0002, true), - "different seeds produced identical shell-on schedule" + + assert!( + oracle::drive_to_quiesce(&mut sim, &mut wl, 20_000), + "{}", + oracle::quiesce_failure_report(&sim, &wl), ); + oracle::assert_converged(&sim, &mut wl); } - /// Drive a full dispatch-shell round-trip for `seed`: seed a partition - /// plus its metadata, log a client in against root, produce one message, - /// then poll. Returns the poll reply's raw bytes and the schedule hash. - fn shell_produce_poll(seed: u64) -> (Vec, u64) { + /// A result-framed transport rejection is not a committed result. + /// + /// Dispatch refuses a request it cannot place (not the primary, transferring, + /// queue full, a view change canceled the pending prepare) and answers with the + /// reason in the reply's RESULT section, under the request's own operation, leaving + /// `status` at 0. That reaches `on_reply` shaped exactly like a commit while + /// carrying a code no op's result enum declares, so the classifier called it a + /// server bug and every fault-injected shell run died on the first one. + /// + /// `TransientNotCommitted` also leaves the outcome UNKNOWN, so the request is held + /// outstanding for a replay, and the quiesce below proves that replay settles + /// rather than stalling the drain. + /// + /// Asserts a transient was seen, so this cannot pass on a path that never produces + /// one. + #[test] + fn shell_workload_survives_result_framed_transient_rejections() { + use crate::workload::{ + FaultInjector, Workload, + options::{ActionWeights, WorkloadOptions}, + oracle, run_with_faults, + }; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), bucket_capacity: 1, }); + let replica_count: u8 = 3; let client_id: u128 = 1; + // Hand-picked: of seeds 1..=40 under these options, ten reached a transient. + // Crashing the primary under a lossy network is necessary but not sufficient, + // so the seed cannot be arbitrary. Re-scan if the PRNG streams are remapped. + let seed = 2; let network_opts = packet::PacketSimulatorOptions { - node_count: 3, + node_count: replica_count, client_count: 1, seed, + packet_loss_probability: 0.10, + replay_probability: 0.03, + one_way_delay_mean: 8, + partition_probability: 0.02, + unpartition_probability: 0.02, + partition_stability: 50, + unpartition_stability: 50, + partition_mode: packet::PartitionMode::UniformSize, + partition_symmetry: packet::PartitionSymmetry::Asymmetric, + path_clog_probability: 0.01, + path_clog_duration_mean: 25, ..packet::PacketSimulatorOptions::default() }; - let mut sim = Simulator::with_shards_shell(3, 1, std::iter::once(client_id), network_opts); - let ns = IggyNamespace::new(0, 0, 0); + let mut sim = Simulator::with_shards_shell( + usize::from(replica_count), + 1, + std::iter::once(client_id), + network_opts, + ); + let ns = IggyNamespace::new(1, 1, 0); sim.init_partition(ns); sim.seed_stream_topic_partition(ns); let client = SimClient::new(client_id); sim.shell_login(&client); - // Produce through the shell too: `SimClient` now emits the legacy - // `SendMessagesHeader` wire shape the real SDK sends, so - // `resolve_partition_request_namespace` decodes it on the - // `handle_client_request` path. Both the write and the poll below now - // exercise the real dispatch layer. - let payload = Bytes::from_static(b"shell-poll-payload"); - let produce = client.send_messages(ns, std::slice::from_ref(&payload)); - sim.submit_request(client_id, 0, produce.into_generic()); - for _ in 0..200 { - sim.step(); - } + let mut options = WorkloadOptions::new(seed, replica_count, vec![ns]); + options.weights = ActionWeights::uniform(); + // Crashing the primary is what puts a live request on a replica that + // cannot place it, which is where the transient comes from. + options.crash_per_tick_ratio = 0.05; + options.restart_per_tick_ratio = 0.08; + options.spare_primary = false; + let mut wl = Workload::new(options); - // Poll through the dispatch shell: on_client_request -> drain -> - // handle_poll_messages -> partition_read -> on_partition_read, running - // as a task the executor interleaves with the pump. - let poll = client.poll_messages(ns, 10); - sim.submit_request(client_id, 0, poll.into_generic()); - let mut poll_reply = None; - for _ in 0..200 { - if let Some(reply) = sim.step().into_iter().next() { - poll_reply = Some(reply); - break; - } - } - let poll_reply = poll_reply.expect("shell poll: no reply within 200 steps"); - (poll_reply.as_slice().to_vec(), sim.schedule_hash()) - } + let clients = [client]; + let mut injector = FaultInjector::new(seed, replica_count); + run_with_faults(&mut sim, &mut wl, &clients, 1_500, u64::MAX, &mut injector); - /// A `SimClient` poll returns the produced messages through the real - /// dispatch read path (`on_client_request` -> `handle_poll_messages` -> - /// `partition_read` -> `on_partition_read`), which runs as a task the - /// executor interleaves with the pump, and the whole login/produce/poll - /// round-trip replays byte-for-byte under one seed. - #[test] - fn shell_poll_returns_produced_messages_deterministically() { - const PAYLOAD: &[u8] = b"shell-poll-payload"; - let (reply_a, schedule_a) = shell_produce_poll(0x5CED_0011); - let (reply_b, schedule_b) = shell_produce_poll(0x5CED_0011); assert!( - reply_a - .windows(PAYLOAD.len()) - .any(|window| window == PAYLOAD), - "poll reply did not carry the produced payload through the real read handler" + wl.auditor.stats().transient_rejections > 0, + "no request was answered with a result-framed transient rejection, so the \ + path this test exists to cover never ran" ); + assert!( - reply_b - .windows(PAYLOAD.len()) - .any(|window| window == PAYLOAD), - "second run's poll reply did not carry the produced payload" - ); - // The produce stamps a random UUID per message (`random_id::get_uuid`), - // so the reply bytes differ run-to-run; the executor schedule is seeded - // and must replay. This mirrors the workload tests, which hash reply - // headers rather than message bodies for the same reason. - assert_eq!( - schedule_a, schedule_b, - "shell schedule diverged at same seed" + oracle::drive_to_quiesce(&mut sim, &mut wl, 50_000), + "{}", + oracle::quiesce_failure_report(&sim, &wl), ); } - /// The dispatch shell's reason to exist: detect the PR #3557 - /// async-concurrency class (a partition reference held across an `.await` - /// while a sibling task mutates the partitions vec). Under the - /// deterministic executor a parked read with a live borrow IS a - /// borrow-held-across-a-suspension, so a concurrent `remove` trips the - /// `#[cfg(debug_assertions)]` borrow tripwire. The correct `with_partition` - /// read drops the borrow before the suspension, so the same interleaving is - /// sound. Debug-only: the tripwire (and this detector) compile out in - /// release, exactly like the `BorrowGuard` they ride on. - /// - /// The fault is injected via the synthetic `hold_borrow_across_await`, not - /// the real read, on purpose: the production read has no borrow-holding - /// suspension to seed. The partition journal read is synchronous (a pure - /// memory copy that never awaits) and `with_partition` returns an owned - /// `PollPlan` before the only awaits (disk read, offset persist) run off the - /// borrow in `spawn_poll_io`, so the real read is sound by construction. - /// - /// TODO: once the simulator models storage faults, the disk-tier read - /// (`PollPlan::execute` -> `read_disk`) becomes a real, seedable await in the - /// read path. A regression holding a partition borrow across it, run against - /// a concurrent reconcile `InsertOwned` reallocation, would then trip this - /// detector end-to-end through the real handler, retiring the synthetic seam. - #[cfg(debug_assertions)] + /// The cross-replica equality check actually compares replicas against each + /// other, and holds over a metadata workload with crashes and restarts. + /// + /// Non-vacuity is the point of the chain assertions. An equality oracle that never + /// finds two replicas at the same op passes in silence, so `ops_compared` counts + /// only ops witnessed on more than one replica, the subset that exercised the + /// property. #[test] - fn shell_detects_partition_borrow_held_across_await() { - use crate::executor::DetExecutor; - use consensus::PartitionsHandle; - use std::panic::{AssertUnwindSafe, catch_unwind}; - + fn committed_metadata_agrees_across_replicas() { + use crate::workload::{ + self, FaultInjector, Workload, + invariants::Invariants, + options::{ActionWeights, WorkloadOptions}, + oracle, + }; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), bucket_capacity: 1, }); + let replica_count: u8 = 5; + let client_id: u128 = 1; + let seed = 0x57A7_E000; let network_opts = packet::PacketSimulatorOptions { - node_count: 3, + node_count: replica_count, client_count: 1, - seed: 0x5CED_0021, + seed, ..packet::PacketSimulatorOptions::default() }; - let mut sim = Simulator::new(3, std::iter::once(1u128), network_opts); - let ns_a = IggyNamespace::new(0, 0, 0); - let ns_b = IggyNamespace::new(0, 0, 1); - sim.init_partition(ns_a); - sim.init_partition(ns_b); + let mut sim = Simulator::new( + usize::from(replica_count), + std::iter::once(client_id), + network_opts, + ); + let client = SimClient::new(client_id); + let ns = IggyNamespace::new(1, 1, 0); + sim.init_partition(ns); + sim.register_client_with_primary(&client); + + // Metadata ops, since the committed chain this checks is the metadata WAL. + // Crash and restart so replicas rejoin and repair, which is when a + // divergence would be introduced if one could be. + let mut options = WorkloadOptions::new(seed, replica_count, vec![ns]); + options.weights = ActionWeights::metadata_only(); + options.crash_per_tick_ratio = 0.01; + options.restart_per_tick_ratio = 0.02; + let mut wl = Workload::new(options); + + let clients = [client]; + let mut injector = FaultInjector::new(seed, replica_count); + let mut invariants = Invariants::new(); + // Driven here rather than through `workload::run` so the accumulated + // chain is readable afterwards; `run` builds its own `Invariants`. + for _ in 0..4_000u32 { + wl.tick(); + injector.step(&mut sim, &wl); + workload::resubmit_due(&mut sim, &mut wl); + if let Some((target, msg)) = wl.build_request(&clients[0]) { + sim.submit_request(clients[0].client_id(), target, msg.into_generic()); + } + for reply in sim.step() { + let cmds = wl.on_reply(&reply); + workload::apply_sim_commands(&mut sim, &cmds); + } + invariants.check(&sim, &wl); + } - // BAD read: holds a partition borrow across a suspension. The mutator - // task runs while it is parked (borrow live) -> the tripwire fires. - // `catch_unwind` builds the executor inline so unwinding drops the - // parked read's guard, restoring the borrow count for the next case. - let tripped = catch_unwind(AssertUnwindSafe(|| { - let mut executor = DetExecutor::new(7); - let read = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - read.plane - .partitions() - .hold_borrow_across_await(std::future::pending()) - .await; - }); - executor.run_until_stalled(POLL_BUDGET); // borrow acquired; task parks - let mutate = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - mutate.plane.partitions().remove(&ns_b); - }); - executor.run_until_stalled(POLL_BUDGET); // mutate while borrow live - })) - .is_err(); assert!( - tripped, - "borrow-held-across-await went undetected: the concurrent mutation \ - did not trip the #3557 borrow tripwire under the executor" + injector.restarts() > 0, + "no replica restarted, so rejoin and repair never ran" ); - // Contiguity: the tripwire fires BEFORE `remove` touches the vec (the - // assert is its first statement), so the detector aborts the mutation - // that would have dangled the live borrow. Both partitions survive - // intact -- the class is caught before it can corrupt state. + let chain = invariants.state_checker(); assert!( - sim.offsets(0, ns_a).is_some() && sim.offsets(0, ns_b).is_some(), - "tripwire must abort the mutation before it corrupts the partitions vec" + chain.chain_len() > 0, + "the canonical commit chain is empty: nothing was ever recorded" + ); + assert!( + chain.ops_compared() > 0, + "no committed op was witnessed on two replicas, so the equality check \ + never actually compared anything and would pass on a diverged cluster" ); - // REAL read: `with_partition` scopes the borrow, dropping it before the - // suspension, so the identical interleaving is sound (no tripwire). - let mut executor = DetExecutor::new(7); - let read = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - let _ = read - .plane - .partitions() - .with_partition(&ns_a, |_partition| ()); - std::future::pending::<()>().await; - }); - executor.run_until_stalled(POLL_BUDGET); - let mutate = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - mutate.plane.partitions().remove(&ns_b); - }); - executor.run_until_stalled(POLL_BUDGET); - // The very mutation the bad read's tripwire aborted now applies - // cleanly (no borrow was live across the suspension): `ns_b` is gone, - // `ns_a` intact -- the correct read is sound under the same schedule. assert!( - sim.offsets(0, ns_a).is_some() && sim.offsets(0, ns_b).is_none(), - "correct with_partition read must leave the concurrent remove sound" + oracle::drive_to_quiesce(&mut sim, &mut wl, 50_000), + "{}", + oracle::quiesce_failure_report(&sim, &wl), ); + assert!( + oracle::settle_to_stable_view(&mut sim, &mut wl, 50_000), + "metadata views never converged after the drain" + ); + oracle::assert_converged(&sim, &mut wl); } - /// The realloc half of the PR #3557 class, and the stronger one: a pump - /// `insert` that grows the partitions vec MOVES every element, so a stale - /// reference to any partition dangles, not just the one a `swap_remove` - /// displaced. `shell_detects_partition_borrow_held_across_await` covers the - /// remove; this covers the grow, on the same two-task deterministic - /// interleave (reconciler-shaped reader parked with a borrow live, pump-shaped - /// task mutating the container underneath it). + /// A lost `PrepareOk` does not wedge the metadata plane: once the acks flow + /// again the primary reaches its commit quorum without client involvement. /// - /// The buffer address is asserted to have MOVED, so the test cannot pass on a - /// push into spare capacity, which relocates nothing. + /// The retransmit is the whole mechanism, and it depends on the duplicate being + /// admitted rather than dropped as a gap: `on_replicate`'s "journal already holds + /// prepare" branch re-forwards it down the chain and re-acks, regenerating the ack + /// the primary lost. Fails if a duplicate ever starts falling through to the gap + /// check instead. /// - /// Debug-only, like the tripwire it drives: `BorrowGuard` compiles out in - /// release. - #[cfg(debug_assertions)] + /// Drops acks rather than crashing anyone, so the property is about lost acks in + /// general, not restart recovery. #[test] - fn shell_detects_partition_borrow_held_across_a_pump_realloc() { - use crate::executor::DetExecutor; - use consensus::PartitionsHandle; - use std::panic::{AssertUnwindSafe, catch_unwind}; + fn lost_prepare_ok_is_recovered_by_retransmit() { + use iggy_binary_protocol::Command; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, @@ -2890,97 +3151,92 @@ mod tests { bucket_capacity: 1, }); + let replica_count: u8 = 3; + let client_id: u128 = 1; let network_opts = packet::PacketSimulatorOptions { - node_count: 3, + node_count: replica_count, client_count: 1, - seed: 0x5CED_0022, + seed: 0x5EED_0077, ..packet::PacketSimulatorOptions::default() }; - let mut sim = Simulator::new(3, std::iter::once(1u128), network_opts); - let ns_a = IggyNamespace::new(0, 0, 0); - let ns_b = IggyNamespace::new(0, 0, 1); - // The namespace the pump grows the vec with. Never materialised up front: - // inserting it IS the mutation under test. - let ns_grow = IggyNamespace::new(0, 0, 2); - sim.init_partition(ns_a); - sim.init_partition(ns_b); - - // BAD read: the borrow is live across the suspension, so the pump's - // growing insert lands while a stale reference to every partition is - // outstanding. `catch_unwind` builds the executor inline so unwinding - // drops the parked read's guard and restores the borrow count. - let tripped = catch_unwind(AssertUnwindSafe(|| { - let mut executor = DetExecutor::new(11); - let read = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - read.plane - .partitions() - .hold_borrow_across_await(std::future::pending()) - .await; - }); - executor.run_until_stalled(POLL_BUDGET); // borrow acquired; task parks - let grow = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - grow.init_partition(ns_grow, None, None); - }); - executor.run_until_stalled(POLL_BUDGET); // grow while the borrow is live - })) - .is_err(); - assert!( - tripped, - "a pump realloc under a live partition borrow went undetected: the \ - #3557 tripwire did not fire on the growing insert" + let mut sim = Simulator::new( + usize::from(replica_count), + std::iter::once(client_id), + network_opts, ); - // The tripwire asserts before `push`, so the vec is untouched: the two - // originals survive and the grow namespace never materialised. - let partitions = sim.replicas[0].shards[0].plane.partitions(); + let client = SimClient::new(client_id); + sim.register_client_with_primary(&client); + + let committed_before = metadata_commit(&sim, 0); + + // Drop only PrepareOk on both backup links. Everything else still flows, + // so the backups receive and journal the prepare; only the primary's + // evidence of that is lost. + for backup in 1..replica_count { + sim.network + .link_filter_mut(ProcessId::Replica(backup), ProcessId::Replica(0)) + .remove(Command::PrepareOk); + } + + let msg = client.create_stream("wl-lost-ack"); + sim.submit_request(client_id, 0, msg.into_generic()); + + // Long enough for the prepare to reach and be journaled by both backups + // while the primary sees no acks. + for _ in 0..200 { + sim.step(); + } assert_eq!( - partitions.len(), - 2, - "tripwire must abort the insert before it relocates the vec" + metadata_commit(&sim, 0), + committed_before, + "the primary must not commit while every backup ack is dropped" ); - assert!(!partitions.contains(&ns_grow)); + for backup in 1..replica_count { + assert!( + metadata_op(&sim, usize::from(backup)) > committed_before, + "backup {backup} must have journaled the prepare, else this test \ + proves nothing about a LOST ack" + ); + } - // REAL read: `with_partition` drops the borrow before the suspension, so - // the identical schedule is sound and the grow applies. - let addr_before = partitions.buffer_addr(); - let mut executor = DetExecutor::new(11); - let read = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - let _ = read - .plane - .partitions() - .with_partition(&ns_a, |_partition| ()); - std::future::pending::<()>().await; - }); - executor.run_until_stalled(POLL_BUDGET); - let grow = Rc::clone(&sim.replicas[0].shards[0]); - executor.spawn(async move { - grow.init_partition(ns_grow, None, None); - }); - executor.run_until_stalled(POLL_BUDGET); + // Restore the acks. From here the primary's retransmit is the only route + // to a commit, which is exactly the mechanism under test. + for backup in 1..replica_count { + sim.network + .link_filter_mut(ProcessId::Replica(backup), ProcessId::Replica(0)) + .insert(Command::PrepareOk); + } - let partitions = sim.replicas[0].shards[0].plane.partitions(); - assert!( - partitions.contains(&ns_grow), - "correct with_partition read must leave the concurrent insert sound" - ); - assert_ne!( - partitions.buffer_addr(), - addr_before, - "the insert landed in spare capacity, so nothing moved and this test \ - proves nothing about a realloc; seed more partitions before the grow" + for _ in 0..5_000 { + sim.step(); + if metadata_commit(&sim, 0) > committed_before { + return; + } + } + panic!( + "metadata commit stuck at {} after 5000 ticks with healthy links: a lost \ + PrepareOk is no longer recovered, so the backup's gap check is now \ + swallowing the primary's retransmit", + metadata_commit(&sim, 0), ); - // Every pre-existing partition is still addressable after the move, which - // is what a stale reference would have missed. - assert!(partitions.contains(&ns_a) && partitions.contains(&ns_b)); } - /// Committed metadata prepare timestamps for `seed`: register plus two - /// stream creates, read back from replica 0's metadata journal. - fn metadata_prepare_timestamps(seed: u64) -> Vec { - use consensus::MetadataHandle; - use journal::{Journal, JournalHandle}; + /// A prepare that every backup journaled but never acked still commits after + /// those backups restart. + /// + /// The rejoin path recovers it: a restarted replica comes back with `current_op` + /// at N from its own WAL, rejoins as a probing backup (`Status::Recovering`, see + /// `new_shard`), and its probe draws a targeted `StartView` that returns it to + /// `Normal` and gets the tail acked. The retransmit cannot do it while the backup + /// is still probing: `replicate_preflight` refuses any prepare outside + /// `Status::Normal`. + /// + /// Pinned because the fuzzer finds runs where this does NOT happen: two live + /// backups hold op 45 with the primary at commit 44, both logging the gap drop for + /// the whole drain. Covering the case that works narrows where that one diverges. + #[test] + fn unacked_prepare_commits_after_the_backup_restarts() { + use iggy_binary_protocol::Command; server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, @@ -2993,85 +3249,100 @@ mod tests { let network_opts = packet::PacketSimulatorOptions { node_count: replica_count, client_count: 1, - seed, + seed: 0x5EED_0078, ..packet::PacketSimulatorOptions::default() }; let mut sim = Simulator::new( - replica_count as usize, + usize::from(replica_count), std::iter::once(client_id), network_opts, ); let client = SimClient::new(client_id); sim.register_client_with_primary(&client); + let committed_before = metadata_commit(&sim, 0); - for name in ["clock-a", "clock-b"] { - let msg = client.create_stream(name); - sim.submit_request(client_id, 0, msg.into_generic()); - let mut got_reply = false; - for _ in 0..100 { - if !sim.step().is_empty() { - got_reply = true; - break; - } + // Lose every backup ack, so the prepare is journaled cluster-wide while + // the primary stays one short of its commit quorum. + for backup in 1..replica_count { + sim.network + .link_filter_mut(ProcessId::Replica(backup), ProcessId::Replica(0)) + .remove(Command::PrepareOk); + } + + let msg = client.create_stream("wl-unacked"); + sim.submit_request(client_id, 0, msg.into_generic()); + for _ in 0..200 { + sim.step(); + } + for backup in 1..replica_count { + assert!( + metadata_op(&sim, usize::from(backup)) > committed_before, + "backup {backup} must hold the prepare before it is restarted" + ); + } + assert_eq!( + metadata_commit(&sim, 0), + committed_before, + "the primary must not have committed while its acks were dropped" + ); + + // Restart every backup. Each recovers the unacked op from its own WAL and + // rejoins as a probing backup, which is the state the primary's + // retransmit cannot get an ack out of. + for backup in 1..replica_count { + sim.replica_crash(backup); + for _ in 0..50 { + sim.tick(); + } + sim.replica_restart(backup); + } + + // Healthy links from here: nothing but the protocol stands between the + // primary and its quorum. + for backup in 1..replica_count { + sim.network + .link_filter_mut(ProcessId::Replica(backup), ProcessId::Replica(0)) + .insert(Command::PrepareOk); + } + + for _ in 0..10_000 { + sim.step(); + if metadata_commit(&sim, 0) > committed_before { + return; } - assert!(got_reply, "create_stream({name}) must commit"); } + panic!( + "metadata commit stuck at {} after 10000 ticks with healthy links and \ + every replica holding op {}: the restarted backups never re-acked the \ + prepare they recovered from their own WALs", + metadata_commit(&sim, 0), + metadata_op(&sim, 1), + ); + } - let shard = &sim.replicas[0].shards[0]; - let journal = shard + /// Committed metadata op on a replica's shard 0. + fn metadata_commit(sim: &Simulator, replica_idx: usize) -> u64 { + sim.replicas[replica_idx].shards[0] .plane .metadata() - .journal + .consensus .as_ref() - .expect("shard 0 owns the metadata journal"); - // Ops 1..=3: Register, then the two creates. - (1..=3) - .map(|op| { - journal - .handle() - .header(op) - .expect("committed op must have a journal header") - .timestamp - }) - .collect() + .expect("shard 0 owns metadata consensus") + .commit_min() } - /// With the injected [`SimClock`], primary-stamped prepare timestamps - /// are a pure function of the seed: identical across same-seed runs, - /// anchored at the synthetic sim epoch (not 1970, not wall clock), - /// and strictly monotonic per the clamp in - /// `next_monotonic_timestamp`. - #[test] - fn prepare_timestamps_replay_with_seed() { - let first = metadata_prepare_timestamps(0xC10C_0001); - let second = metadata_prepare_timestamps(0xC10C_0001); - assert_eq!( - first, second, - "prepare timestamps diverged across same-seed runs" - ); - for timestamp in &first { - assert!( - *timestamp >= deps::SIM_EPOCH_MICROS, - "timestamp {timestamp} predates the sim epoch; wall clock leaked" - ); - // Sim runs complete in well under a simulated day; a wall-clock - // leak would stamp 2026-07+ values far past this bound. - assert!( - *timestamp < deps::SIM_EPOCH_MICROS + 86_400_000_000, - "timestamp {timestamp} beyond epoch + 1 day; wall clock leaked" - ); - } - assert!( - first.windows(2).all(|pair| pair[0] < pair[1]), - "prepare timestamps must be strictly monotonic: {first:?}" - ); + /// Highest metadata op a replica has journaled. + fn metadata_op(sim: &Simulator, replica_idx: usize) -> u64 { + sim.replicas[replica_idx] + .metadata_journal + .last_op() + .unwrap_or(0) } - /// IGGY-66 acceptance: per-partition consensus independence. Block - /// `ns_a`'s `PrepareOk` acks at the network layer and fill its - /// pipeline to `PIPELINE_PREPARE_QUEUE_MAX`; a request on `ns_b` - /// still commits while `ns_a` is wedged (no quorum without backup - /// acks); lifting the block drains `ns_a` completely. + /// IGGY-66 acceptance: per-partition consensus independence. Blocking `ns_a`'s + /// `PrepareOk` acks and filling its pipeline to `PIPELINE_PREPARE_QUEUE_MAX` + /// wedges it for want of quorum, `ns_b` still commits, and lifting the block + /// drains `ns_a` completely. #[test] fn per_partition_consensus_independence() { use consensus::PIPELINE_PREPARE_QUEUE_MAX; @@ -3238,7 +3509,7 @@ mod tests { oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000), "system did not drain within the tick budget" ); - oracle::assert_converged(&sim, &wl); + oracle::assert_converged(&sim, &mut wl); assert_no_frame_drops(&sim); } } @@ -3250,20 +3521,18 @@ mod view_change_data_loss_tests { //! //! Without the sender's log suffix on the `DoViewChange`, the new primary adopts //! the winner's op NUMBER, rebuilds its pipeline from its OWN journal, hits the - //! hole, and truncates the range as "decided lost" -- discarding an op journaled - //! on a quorum and already replied to. The next client op then reuses the number - //! and collides with the stale entry on the up-to-date backup. + //! hole and truncates the range as "decided lost", discarding an op journaled on a + //! quorum and already replied to. The next client op reuses that number and + //! collides with the stale entry on the up-to-date backup. //! - //! The hole here is punched at the commit point, so the assertion that catches a - //! regression is "the op came back", not "the head did not regress": with nothing - //! uncommitted there is no pipeline rebuild to truncate. The `dvc_merge` unit - //! tests cover the sequencer-truncation path directly. + //! The hole is punched at the commit point, so the regression is caught by "the op + //! came back" rather than "the head did not regress": with nothing uncommitted + //! there is no pipeline rebuild to truncate. The `dvc_merge` unit tests cover the + //! sequencer-truncation path directly. use super::*; - use crate::executor::yield_once; use consensus::{Sequencer, Status}; use journal::Journal; - use message_bus::MessageBus; /// Whether a replica's shard-0 metadata consensus is a settled primary in a /// view past the one that crashed. @@ -3402,131 +3671,21 @@ mod view_change_data_loss_tests { "op {committed} must be repaired back into the new primary's journal" ); } +} - /// A client submit landing inside the new primary's view-start superblock - /// persist must not corrupt the pipeline. - /// - /// `start_pending_view` flips the replica into a Normal primary - /// synchronously and defers the rebuild of the inherited uncommitted - /// suffix; the persist then suspends the pump. A register admitted in that - /// window used to mint the next op into the still-empty pipeline, and the - /// deferred rebuild panicked pushing the inherited op beneath it - /// ("sequence must be sequential"); the same empty pipeline also blinded - /// the register dedup, admitting an inherited in-flight register twice. - /// The suspension is real on disk-backed stores (an fsync) and is restored - /// here with `set_yield_writes`. - #[test] - fn given_a_register_inside_the_view_start_persist_when_the_pipeline_rebuilds_should_commit_once() - { - server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let replica_count: u8 = 3; - let settled_client: u128 = 1; - let straggler_client: u128 = 2; - let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, - client_count: 2, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new( - replica_count as usize, - [settled_client, straggler_client].into_iter(), - network_opts, - ); - - let client = SimClient::new(settled_client); - sim.register_client_with_primary(&client); - for _ in 0..100 { - sim.step(); - } - let (baseline_head, baseline_commit) = metadata_progress(&sim, 1); - assert_eq!( - baseline_head, baseline_commit, - "the cluster must be quiescent before the straggler is staged" - ); - - // Stage the inherited suffix: the straggler's register reaches the - // next primary's journal, then the old primary dies before the commit - // makes it back. - let straggler = SimClient::new(straggler_client); - sim.submit_request(straggler_client, 0, straggler.register().into_generic()); - let mut staged = None; - for _ in 0..200 { - sim.step(); - let (head, commit_max) = metadata_progress(&sim, 1); - if head > baseline_head && commit_max < head { - staged = Some(head); - break; - } - } - let staged = - staged.expect("the register must reach the next primary's journal before it commits"); - - // Both survivors' next persists suspend once, opening the window a - // real fsync has. - sim.replicas[1].superblock.set_yield_writes(); - sim.replicas[2].superblock.set_yield_writes(); - sim.replica_crash(0); - - // The straggler's retry loop, as the server runs it: `dispatch` spawns - // the in-process submit on its own task, which is what can interleave - // with the parked pump. The sim's wire path processes requests inside - // the pump itself, so the window is only reachable from a spawned - // task. A plain once-per-step retry is never ready inside the drain - // where the pump flips to primary and suspends on the persist, so - // each tick wake spends a small budget of yield-separated attempts: - // the yields land the retry between the pump's polls, one of which is - // the suspended view-start persist. - let registered = std::rc::Rc::new(std::cell::Cell::new(false)); - let submit_shard = std::rc::Rc::clone(&sim.replicas[1].shards[0]); - let submit_flag = std::rc::Rc::clone(®istered); - sim.executor.spawn(async move { - loop { - for _ in 0..32 { - match submit_shard - .plane - .metadata() - .submit_register_in_process(straggler_client, 0) - .await - { - Ok(_) => { - submit_flag.set(true); - return; - } - Err(error) if error.is_transient() => yield_once().await, - Err(_) => return, - } - } - submit_shard - .bus - .sleep(std::time::Duration::from_millis(10)) - .await; - } - }); - - for _ in 0..1500 { - sim.step(); - if registered.get() { - break; - } - } - assert!( - registered.get(), - "the straggler's login must complete after the failover" - ); - - let primary = (1..replica_count) - .find(|&replica| is_new_metadata_primary(&sim, replica)) - .expect("a metadata primary must be elected after the old one crashes"); - let (_, commit_max) = metadata_progress(&sim, primary); - assert!( - commit_max >= staged, - "the inherited op ({staged}) must commit under the new primary \ - (commit_max = {commit_max})" - ); - } +/// Whether a setup-handshake reply is a result-framed transport rejection rather +/// than an answer. `build_result_rejection_reply` carries the reason in the result +/// section under the request's own operation with `status` left at 0, so nothing in +/// the header distinguishes it from a commit and the code is the only signal. +pub(crate) fn setup_reply_is_transient(reply: &Message) -> bool { + let header = reply.header(); + let Some(body) = reply + .as_slice() + .get(size_of::()..header.size as usize) + else { + return false; + }; + iggy_binary_protocol::result_code(body) + .and_then(workload::TransientRejection::from_code) + .is_some() } diff --git a/core/simulator/src/network.rs b/core/simulator/src/network.rs index 30aaa36a33..43984693cb 100644 --- a/core/simulator/src/network.rs +++ b/core/simulator/src/network.rs @@ -22,9 +22,10 @@ //! process-to-bus routing, and node enable/disable logic. use crate::packet::{ - ALLOW_ALL, BLOCK_ALL, LinkFilter, Packet, PacketSimulator, PacketSimulatorOptions, ProcessId, + ALLOW_ALL, BLOCK_ALL, COMMAND_COUNT_MAX, LinkFilter, Packet, PacketSimulator, + PacketSimulatorOptions, ProcessId, }; -use iggy_binary_protocol::GenericHeader; +use iggy_binary_protocol::{Command, GenericHeader}; use server_common::Message; /// Network layer for the cluster simulation. @@ -129,6 +130,11 @@ impl Network { /// /// **Warning:** resets all link filters to [`ALLOW_ALL`], including /// manually-set per-command filters. + /// End fault injection: see [`PacketSimulator::heal`]. + pub fn heal(&mut self) { + self.simulator.heal(); + } + pub fn clear_partition(&mut self) { self.simulator.clear_partition(); } @@ -164,4 +170,17 @@ impl Network { pub fn packets_in_flight(&self) -> usize { self.simulator.packets_in_flight() } + + /// Packets delivered so far, per [`Command`] discriminant. Names come from + /// [`COMMAND_LABELS`](crate::packet::COMMAND_LABELS). + #[must_use] + pub const fn command_counts(&self) -> &[u64; COMMAND_COUNT_MAX] { + self.simulator.command_counts() + } + + /// Whether any packet of this command reached its destination. + #[must_use] + pub const fn delivered_any(&self, command: Command) -> bool { + self.simulator.delivered_any(command) + } } diff --git a/core/simulator/src/packet.rs b/core/simulator/src/packet.rs index 921b035c36..816e2fa23c 100644 --- a/core/simulator/src/packet.rs +++ b/core/simulator/src/packet.rs @@ -39,13 +39,15 @@ //! the same tick cannot trigger chain reactions within that tick. use crate::ready_queue::{Ready, ReadyQueue}; +use crate::seeds::SimSeeds; use enumset::EnumSet; use iggy_binary_protocol::{Command, GenericHeader}; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use rand_xoshiro::rand_core::SeedableRng; use server_common::Message; use std::collections::HashMap; +use strum::{EnumCount, EnumIter, IntoEnumIterator}; /// Per-link command filter. An `EnumSet` where: /// - [`ALLOW_ALL`] = all commands pass (link fully enabled) @@ -153,6 +155,76 @@ impl Default for PacketSimulatorOptions { } } +impl PacketSimulatorOptions { + /// Every network parameter drawn from `seed`, so the seed picks the weather as + /// well as the traffic. + /// + /// A fixed profile explores ONE point in parameter space however many seeds are + /// thrown at it, so `--faults heavy` run a thousand times is the same network a + /// thousand times. + /// + /// `node_count` and `client_count` are left at their defaults for the caller to + /// fill, as [`Self::default`] leaves them; `seed` is stamped here so a value + /// used as-is still replays. + /// + /// The ceilings sit roughly 1.5x above the hand-calibrated `heavy` profile, + /// which already costs an order of magnitude of throughput: far enough to reach + /// past what a fixed profile could, near enough that a healthy cluster still + /// drains and a failure to converge is worth reading. A tick here runs every + /// shard's pump to quiescence rather than one IO step, so the same percentages + /// describe a more hostile network than they would in a per-IO model. Forcing a + /// single axis past its ceiling is what the individual `--packet-loss-prob` + /// overrides are for. + #[must_use] + pub fn swarm(seed: u64) -> Self { + // The swarm stream, not the network one: [`PacketSimulator`] draws its + // delays and drops from the same `seed`, so sharing would correlate the loss + // probability with the loss events it produces. + let mut prng = Xoshiro256PlusPlus::seed_from_u64(SimSeeds::derive(seed).swarm); + // `PacketSimulator::new` asserts `min >= 1` (zero causes unbounded replay + // loops) and `mean >= min`, so both are drawn to satisfy it rather than + // clamped afterwards. + let one_way_delay_min = prng.random_range(1..=3u64); + let one_way_delay_mean = prng.random_range(one_way_delay_min..=10u64); + Self { + one_way_delay_min, + one_way_delay_mean, + packet_loss_probability: f64::from(prng.random_range(0..=15u32)) / 100.0, + replay_probability: f64::from(prng.random_range(0..=5u32)) / 100.0, + // Floored well above 2, deliberately. A tiny queue drops packets by + // eviction, the same fault class as `packet_loss_probability` above, so + // the two compound into a network that never converges without covering + // anything the loss draw does not. `--link-capacity` forces it lower. + link_capacity: prng.random_range(8..=64u8), + partition_probability: f64::from(prng.random_range(0..=30u32)) / 1_000.0, + // Never zero: a partition that cannot heal is a permanently split + // cluster, and every run drawing it reports a liveness failure that + // says nothing. + unpartition_probability: f64::from(prng.random_range(1..=10u32)) / 100.0, + partition_stability: prng.random_range(20..=80u32), + unpartition_stability: prng.random_range(0..=60u32), + partition_mode: draw_variant(&mut prng), + partition_symmetry: draw_variant(&mut prng), + path_clog_probability: f64::from(prng.random_range(0..=15u32)) / 1_000.0, + path_clog_duration_mean: prng.random_range(0..=40u64), + seed, + ..Self::default() + } + } +} + +/// Uniform draw over an enum's variants. +/// +/// Generic rather than a `match` on a drawn index: a match would silently keep +/// drawing the old variant set after one is added to [`PartitionMode`], and a +/// fault mode the swarm never reaches looks like one that never finds anything. +fn draw_variant(prng: &mut Xoshiro256PlusPlus) -> T { + let index = prng.random_range(0..T::COUNT); + T::iter() + .nth(index) + .expect("an index drawn below the variant count always names a variant") +} + /// Per-path link: holds packets in a [`ReadyQueue`] sorted by `ready_at`. struct Link { /// Packets waiting to be delivered, ordered by `ready_at` (min-heap). @@ -192,7 +264,11 @@ impl Link { /// Determines how automatic partitions are created. /// Only nodes (replicas) are partitioned. There will always be exactly two partitions. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +/// +/// `EnumCount` + `EnumIter` so [`PacketSimulatorOptions::swarm`] can draw a +/// variant uniformly; adding one here puts it in the swarm's reach with no +/// second edit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, EnumCount, EnumIter)] pub enum PartitionMode { /// Disable automatic partitioning. #[default] @@ -207,8 +283,9 @@ pub enum PartitionMode { IsolateSingle, } -/// Whether partitions are symmetric or asymmetric. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +/// Whether partitions are symmetric or asymmetric. See [`PartitionMode`] for +/// why the strum derives are here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, EnumCount, EnumIter)] pub enum PartitionSymmetry { #[default] Symmetric, @@ -220,6 +297,13 @@ pub struct PacketSimulator { options: PacketSimulatorOptions, /// Flat array of links. Index = `from_idx` * `max_processes` + `to_idx`. links: Vec, + /// Whether each process is running, indexed by flat process index. + /// + /// A layer ABOVE the link filters, never written into them, so a crash and a + /// partition compose. Folding availability into `Link::filter` meant restarting a + /// process wrote `ALLOW_ALL` over whatever the partition had set. Mirrors + /// TigerBeetle's `buses_enabled`. + process_up: Vec, /// Maximum number of processes (determines link array size). max_processes: usize, /// Mapping from [`ProcessId`] to flat index. @@ -230,7 +314,7 @@ pub struct PacketSimulator { /// Current tick (network global time). current_tick: u64, /// PRNG for deterministic randomness. - prng: Xoshiro256Plus, + prng: Xoshiro256PlusPlus, /// Whether an automatic partition is currently active. auto_partition_active: bool, /// Per-node partition assignment (true = partition A, false = partition B). @@ -241,8 +325,65 @@ pub struct PacketSimulator { auto_partition_nodes: Vec, /// Reusable buffer for delivered packets. delivered: Vec, + /// Packets actually delivered, per [`Command`] discriminant. + /// + /// Counted at delivery, past every drop path, so a command appears only if a + /// process really received one. This is how a run answers which parts of the + /// protocol it exercised, which is the difference between covering a path and + /// merely compiling it. + command_counts: [u64; COMMAND_COUNT_MAX], } +/// One past the highest [`Command`] discriminant, sizing [`COMMAND_LABELS`] and +/// the delivery counters. Raising it is part of adding a command. +pub const COMMAND_COUNT_MAX: usize = 30; + +/// Names for each [`Command`] discriminant, so a coverage report reads as +/// protocol rather than as integers. Indexed by discriminant; the trailing +/// assert keeps it aligned with the enum. +pub const COMMAND_LABELS: [&str; COMMAND_COUNT_MAX] = [ + "Reserved", + "Ping", + "Pong", + "PingClient", + "PongClient", + "Request", + "Prepare", + "PrepareOk", + "Reply", + "Commit", + "StartViewChange", + "DoViewChange", + "StartView", + "Eviction", + "ReplicaHello", + "ReplicaChallenge", + "ReplicaFinish", + "RequestStartView", + "RequestPrepares", + "RepairPrepare", + "RepairDone", + "RangeEvicted", + "RequestStateTransfer", + "StateTransferTarget", + "RequestStateChunk", + "StateChunk", + "ForwardRegister", + "ForwardRegisterResult", + "ForwardLogout", + "ForwardLogoutResult", +]; + +const _: () = { + // Adding a command without extending the table would report it under the wrong + // name or index past the end of `command_counts`. Asserts the TABLE's length + // rather than pinning one variant to the end: `ForwardLogoutResult == + // COMMAND_COUNT_MAX - 1` still holds after a `NewThing = 30` is appended past + // it, so that form passed exactly when it needed to fire. + assert!(COMMAND_LABELS.len() == COMMAND_COUNT_MAX); + assert!(enumset::EnumSet::::variant_count() as usize == COMMAND_COUNT_MAX); +}; + impl PacketSimulator { /// Create a new packet simulator. /// @@ -312,16 +453,18 @@ impl PacketSimulator { Self { options, links, + process_up: vec![true; max_processes], max_processes, process_indices, next_index: node_count, current_tick: 0, - prng: Xoshiro256Plus::seed_from_u64(seed), + prng: Xoshiro256PlusPlus::seed_from_u64(SimSeeds::derive(seed).network), auto_partition_active: false, auto_partition: vec![false; node_count], auto_partition_stability: initial_stability, auto_partition_nodes: (0..node_count).collect(), delivered: Vec::new(), + command_counts: [0; COMMAND_COUNT_MAX], } } @@ -402,7 +545,7 @@ impl PacketSimulator { /// Calculate a random delay using exponential distribution. /// Returns max(min, exponential(mean)). - fn calculate_delay(prng: &mut Xoshiro256Plus, options: &PacketSimulatorOptions) -> u64 { + fn calculate_delay(prng: &mut Xoshiro256PlusPlus, options: &PacketSimulatorOptions) -> u64 { let min = options.one_way_delay_min; let mean = options.one_way_delay_mean; let exp = Self::random_exponential(prng, mean); @@ -416,7 +559,7 @@ impl PacketSimulator { clippy::cast_sign_loss, clippy::cast_possible_truncation )] - fn random_exponential(prng: &mut Xoshiro256Plus, mean: u64) -> u64 { + fn random_exponential(prng: &mut Xoshiro256PlusPlus, mean: u64) -> u64 { let u: f64 = prng.random::(); if u > 0.0 { (-(mean as f64) * u.ln()) as u64 @@ -467,31 +610,31 @@ impl PacketSimulator { &mut self.links[idx].drop_packet_fn } - /// Disable a process by blocking all links to and from it. + /// Mark a process down. Anything addressed to it is dropped at delivery. /// - /// Packets already queued on those links remain but will be dropped at - /// delivery time because the link filter is [`BLOCK_ALL`]. + /// Link filters are untouched, so a partition or command filter standing at crash + /// time still stands at restart. pub fn process_disable(&mut self, process: ProcessId) { - let all_processes: Vec = self.process_indices.keys().copied().collect(); - for other in all_processes { - if other == process { - continue; - } - *self.link_filter(process, other) = BLOCK_ALL; - *self.link_filter(other, process) = BLOCK_ALL; - } + let idx = self + .process_index(process) + .expect("process_disable: unregistered process"); + self.process_up[idx] = false; } - /// Re-enable a process by allowing all links to and from it. + /// Mark a process up again. Restores nothing else: whatever the link layer was + /// applying before the crash still applies after the restart. pub fn process_enable(&mut self, process: ProcessId) { - let all_processes: Vec = self.process_indices.keys().copied().collect(); - for other in all_processes { - if other == process { - continue; - } - *self.link_filter(process, other) = ALLOW_ALL; - *self.link_filter(other, process) = ALLOW_ALL; - } + let idx = self + .process_index(process) + .expect("process_enable: unregistered process"); + self.process_up[idx] = true; + } + + /// Whether a process is currently running. + #[must_use] + pub fn is_process_up(&self, process: ProcessId) -> bool { + self.process_index(process) + .is_some_and(|idx| self.process_up[idx]) } // TODO: implement record/replay_recorded for deterministic replay support. @@ -518,12 +661,14 @@ impl PacketSimulator { let Self { links, + process_up, prng, options, current_tick, delivered, max_processes, next_index, + command_counts, .. } = self; @@ -544,6 +689,14 @@ impl PacketSimulator { break; }; + // Discarded on arrival rather than by blocking the link, which is + // what lets a crash and a partition stand at once. Target only, as + // in TigerBeetle: a packet sent before the sender died still lands. + if !process_up[to] { + tracing::trace!(to, "packet dropped (target process is down)"); + continue; + } + // Per-command link filter check: drop if command not in filter let command = packet.message.header().command; if !link.filter.contains(command) { @@ -580,6 +733,7 @@ impl PacketSimulator { tracing::trace!("packet replayed"); } + command_counts[command as usize] += 1; delivered.push(packet); } } @@ -588,6 +742,20 @@ impl PacketSimulator { std::mem::take(&mut self.delivered) } + /// Packets delivered so far, per [`Command`] discriminant. See + /// [`Self::command_counts`]'s field docs for why this counts at delivery. + #[must_use] + pub const fn command_counts(&self) -> &[u64; COMMAND_COUNT_MAX] { + &self.command_counts + } + + /// Whether any packet of this command has been delivered. The question a + /// coverage assertion actually asks. + #[must_use] + pub const fn delivered_any(&self, command: Command) -> bool { + self.command_counts[command as usize] > 0 + } + /// Return a previously taken buffer for reuse. pub fn recycle_buffer(&mut self, mut buf: Vec) { buf.clear(); @@ -714,6 +882,22 @@ impl PacketSimulator { self.current_tick } + /// End fault injection: heal what is broken and stop drawing new faults. + /// + /// A drain cannot prove convergence while the generator that broke connectivity + /// keeps breaking it. Mirrors TigerBeetle's `transition_to_liveness_mode`. Delays + /// stay: they slow a drain, they do not prevent it. + pub fn heal(&mut self) { + self.options.packet_loss_probability = 0.0; + self.options.replay_probability = 0.0; + self.options.partition_probability = 0.0; + self.options.path_clog_probability = 0.0; + self.clear_partition(); + for link in &mut self.links { + link.clogged_till = 0; + } + } + /// Clear all partitions, restoring full connectivity. /// /// Resets auto partition state and sets **all** link filters to `ALLOW_ALL`, @@ -922,6 +1106,125 @@ mod tests { assert_eq!(delivered.len(), 1); } + /// A partition standing when a process crashes still stands when it restarts. + /// + /// `process_enable` used to write `ALLOW_ALL` over every link touching the + /// restarted process, clearing its share of a partition still reported active. + /// Both symmetries, because a blanket re-enable erases either. + #[test] + fn a_restart_leaves_a_standing_partition_intact() { + for symmetry in [PartitionSymmetry::Symmetric, PartitionSymmetry::Asymmetric] { + let mut sim = PacketSimulator::new(PacketSimulatorOptions { + one_way_delay_min: 1, + one_way_delay_mean: 1, + partition_probability: 1.0, + unpartition_probability: 0.0, + partition_stability: 1_000, + unpartition_stability: 0, + partition_mode: PartitionMode::UniformSize, + partition_symmetry: symmetry, + node_count: 3, + client_count: 0, + seed: 0x9A11, + ..Default::default() + }); + + sim.tick(); + assert!( + sim.auto_partition_active, + "{symmetry:?}: expected a partition" + ); + let filters_while_partitioned: Vec = + sim.links.iter().map(|link| link.filter).collect(); + assert!( + filters_while_partitioned.iter().any(EnumSet::is_empty), + "{symmetry:?}: the partition blocked no link, so this proves nothing" + ); + + sim.process_disable(ProcessId::Replica(0)); + assert!(!sim.is_process_up(ProcessId::Replica(0))); + sim.process_enable(ProcessId::Replica(0)); + assert!(sim.is_process_up(ProcessId::Replica(0))); + + let filters_after_restart: Vec = + sim.links.iter().map(|link| link.filter).collect(); + assert_eq!( + filters_after_restart, filters_while_partitioned, + "{symmetry:?}: restarting a replica changed the partition's link state" + ); + assert!( + sim.auto_partition_active, + "{symmetry:?}: the partition must still be active after the restart" + ); + } + } + + /// A hand-set per-command filter survives a crash and restart. + /// + /// A restart restoring it turns a scenario test's targeted fault into no fault at + /// all, with the test still green. + #[test] + fn a_restart_leaves_a_manual_command_filter_intact() { + let mut sim = PacketSimulator::new(PacketSimulatorOptions { + one_way_delay_min: 1, + one_way_delay_mean: 1, + node_count: 2, + client_count: 0, + seed: 0x9A12, + ..Default::default() + }); + + let from = ProcessId::Replica(0); + let to = ProcessId::Replica(1); + let filter = ALLOW_ALL - Command::Prepare; + *sim.link_filter(from, to) = filter; + + sim.process_disable(to); + sim.process_enable(to); + + assert_eq!( + *sim.link_filter(from, to), + filter, + "the restart restored a command the filter was dropping" + ); + } + + /// Packets addressed to a crashed process are dropped on arrival; the process + /// receives again the moment it is back. What the availability layer owes now that + /// link filters no longer carry it. + #[test] + fn a_down_process_receives_nothing_and_recovers_on_restart() { + let mut sim = PacketSimulator::new(PacketSimulatorOptions { + one_way_delay_min: 1, + one_way_delay_mean: 1, + node_count: 2, + client_count: 0, + seed: 0x9A13, + ..Default::default() + }); + + let from = ProcessId::Replica(0); + let to = ProcessId::Replica(1); + + // Exponential delays, so drain over a window rather than a single tick. + let drain = |sim: &mut PacketSimulator| { + let mut delivered = 0; + for _ in 0..50 { + sim.tick(); + delivered += sim.step().len(); + } + delivered + }; + + sim.process_disable(to); + sim.submit(from, to, create_test_message_with_command(Command::Prepare)); + assert_eq!(drain(&mut sim), 0, "a crashed replica must receive nothing"); + + sim.process_enable(to); + sim.submit(from, to, create_test_message_with_command(Command::Prepare)); + assert_eq!(drain(&mut sim), 1, "a restarted replica must receive again"); + } + #[test] fn test_auto_partition_lifecycle() { let options = PacketSimulatorOptions { diff --git a/core/simulator/src/ready_queue.rs b/core/simulator/src/ready_queue.rs index 8deb1bc057..a9dd189249 100644 --- a/core/simulator/src/ready_queue.rs +++ b/core/simulator/src/ready_queue.rs @@ -229,9 +229,9 @@ mod tests { } } - fn make_prng() -> rand_xoshiro::Xoshiro256Plus { + fn make_prng() -> rand_xoshiro::Xoshiro256PlusPlus { use rand_xoshiro::rand_core::SeedableRng; - rand_xoshiro::Xoshiro256Plus::seed_from_u64(42) + rand_xoshiro::Xoshiro256PlusPlus::seed_from_u64(42) } #[test] diff --git a/core/simulator/src/replica.rs b/core/simulator/src/replica.rs index cbcffc6712..08e64ad992 100644 --- a/core/simulator/src/replica.rs +++ b/core/simulator/src/replica.rs @@ -20,10 +20,13 @@ use crate::deps::SimSuperblock; use crate::deps::{MemStorage, SimJournal, SimMuxStateMachine, SimSnapshot}; use configs::server::PersonalAccessTokenConfig; use configs::server::ServerSystemConfig; -use consensus::{ConsensusClock, LocalPipeline, Sequencer, VsrConsensus, VsrState}; +use consensus::{ClientTable, ConsensusClock, LocalPipeline, Sequencer, VsrConsensus, VsrState}; use iggy_common::IggyByteSize; use iggy_common::variadic; +use journal::Journal; +use metadata::impls::metadata::IggySnapshot; use metadata::stm::mux::WithFactory; +use metadata::stm::snapshot::RestoreSnapshot; use metadata::stm::stream::{Streams, StreamsInner}; use metadata::stm::user::{Users, UsersInner}; use metadata::{IggyMetadata, apply_committed_prepare}; @@ -82,6 +85,30 @@ pub type SimMetadataBundle = ::Bundle; /// metrics stay zero. pub const SIM_INBOX_CAPACITY: usize = 8192; +/// Read this replica's persisted metadata checkpoint and its trailer checksum. +/// +/// A file that exists but does not load panics rather than reading as `None`: +/// production refuses boot on a torn snapshot, and booting empty instead would turn +/// a storage fault into the silent state loss this path exists to prevent. +/// +/// # Panics +/// If `snapshot.bin` exists but cannot be read or decoded. +fn load_local_checkpoint(data_dir: &std::path::Path) -> Option<(IggySnapshot, u128)> { + let path = data_dir + .join(metadata::impls::METADATA_DIR) + .join(metadata::impls::SNAPSHOT_FILE_NAME); + if !path.exists() { + return None; + } + let loaded = IggySnapshot::load(&path).unwrap_or_else(|error| { + panic!( + "metadata checkpoint at {} exists but does not load: {error}", + path.display(), + ) + }); + Some(loaded) +} + /// Build one shard of a sim replica around the real inter-shard mesh. /// /// `senders`/`inbox` come from [`shard::shard_mesh_channels`], so the @@ -126,6 +153,7 @@ pub fn new_shard( metadata_journal: Option>>, recovered_state: Option, incarnation: u128, + data_dir: Option, ) -> (Rc, Option) { // Metadata is single-writer, mirroring the server bootstrap. Shard 0 owns // the only writable STM; every peer shard rebuilds a reader-mode mirror from @@ -146,6 +174,20 @@ pub fn new_shard( let restored_op = metadata_journal .as_ref() .and_then(|journal| journal.last_op()); + // Read before the state machine is built: it REPLACES the seeded default rather + // than layering onto it, so without it a checkpointed replica recovers an empty + // baseline and loses every op the drain reclaimed. + let local_checkpoint = data_dir + .as_deref() + .filter(|_| shard_idx == 0) + .and_then(load_local_checkpoint); + let has_checkpoint = local_checkpoint.is_some(); + // Off by one from the journal's watermark, deliberately: a checkpoint at op N + // drains `0..=N - 1` and retains N's header for view-change merging, so replaying + // from the journal floor would re-apply N on top of a snapshot that has it. + let checkpoint_seq = local_checkpoint + .as_ref() + .map_or(0, |(snapshot, _)| snapshot.snapshot().sequence_number); let mux = reader_bundle.map_or_else( // Writer shard (shard 0). Seed the root user at slab id 0, matching @@ -160,6 +202,12 @@ pub fn new_shard( // after `IggyMetadata` is built (see below), so the factory bundle is minted // only then. || { + // Seeding on top of a restored checkpoint would mint a second root user + // and shift every workload entity's slab id. + if let Some((snapshot, _)) = local_checkpoint.as_ref() { + return SimMuxStateMachine::restore_snapshot(snapshot.snapshot()) + .expect("the local checkpoint decodes into the metadata state machine"); + } let users: Users = UsersInner::new().into(); let root_password_hash = if shell { crypto::hash_with_fixed_salt(SHELL_ROOT_PASSWORD) @@ -219,14 +267,14 @@ pub fn new_shard( // of an uncommitted suffix is skipped, since a rejoining replica is always a // backup. let restored_head = restored_op.unwrap_or(0); - if !solo && (restored_head > 0 || recovered_state.is_some()) { + if !solo && (restored_head > 0 || recovered_state.is_some() || has_checkpoint) { consensus.init_as_backup(); consensus.begin_view_probe(); } else { consensus.init(); } if let (Some(journal), Some(head)) = (metadata_journal.as_ref(), restored_op) { - let commit_watermark = journal.recovery_commit_watermark(solo); + let commit_watermark = journal.recovery_commit_watermark(solo).max(checkpoint_seq); consensus.sequencer().set_sequence(head); consensus.restore_commit_state(commit_watermark, commit_watermark); if let Some(header) = last_header { @@ -244,15 +292,33 @@ pub fn new_shard( }); let metadata_snapshot = (shard_idx == 0).then(SimSnapshot::default); + // A data directory arms the `SnapshotCoordinator`; without one + // `checkpoint_if_needed` returns immediately and nothing ever checkpoints. let metadata = IggyMetadata::new( metadata_consensus, metadata_journal, metadata_snapshot, superblock, mux, - None, + data_dir, ); + // Both halves are load-bearing: the pairing keeps a later view-change superblock + // write from regressing to `(0, 0)`, and the folded table is the floor the replayed + // suffix advances, so a session below the drained prefix keeps its watermark. + if let Some((snapshot, checksum)) = local_checkpoint.as_ref() { + metadata.seed_checkpoint_ref(snapshot.snapshot().sequence_number, *checksum); + if let Some(table) = snapshot.snapshot().client_table.clone() { + let capacity = metadata.client_table_capacity(); + let restored = ClientTable::from_snapshot(table, capacity) + .expect("the local checkpoint's client table decodes"); + assert!( + metadata.install_client_table(restored), + "a freshly built plane holds no sessions, so this must install" + ); + } + } + // Reconstruct shard 0's committed metadata from the retained WAL, the sim analog // of production's snapshot + WAL replay. `apply_committed_prepare` is the SAME // apply path the commit walk uses, so it rebuilds BOTH the state machine and the @@ -261,19 +327,31 @@ pub fn new_shard( // watermark; this only rebuilds derived state, and does nothing on a fresh boot. // The commit notifier is unwired during recovery, hence the no-op hook. if let Some(journal) = metadata.journal.as_ref() { - let commit_watermark = journal.recovery_commit_watermark(solo); - for op in 1..=commit_watermark { - if let Some(entry) = journal.entry_sync(op) { - // `true`: the sim performs no state transfer, so no frontier - // shields any op from the table half of the apply. - apply_committed_prepare( - &metadata.mux_stm, - &metadata.client_table, - true, - |_| {}, - entry, + let floor = journal.snapshot_op().max(checkpoint_seq); + let commit_watermark = journal.recovery_commit_watermark(solo).max(checkpoint_seq); + for op in (floor + 1)..=commit_watermark { + let Some(entry) = journal.entry_sync(op) else { + // Applying across a hole would replay effects onto a state machine + // that never saw the missing op; the suffix is left for VSR repair. + tracing::warn!( + replica_id, + op, + floor, + commit_watermark, + "metadata WAL has no entry at this committed op; stopping replay" ); - } + break; + }; + // `true`: the frontier is volatile, so a rebuilt plane starts at zero and + // every op above the floor must mutate the table. A frontier only fences a + // LIVE table a state transfer just replaced. + apply_committed_prepare( + &metadata.mux_stm, + &metadata.client_table, + true, + |_| {}, + entry, + ); } } // Mint the peers' read-side bundle AFTER reconstruction so it reflects the diff --git a/core/simulator/src/seeds.rs b/core/simulator/src/seeds.rs new file mode 100644 index 0000000000..d9791705b2 --- /dev/null +++ b/core/simulator/src/seeds.rs @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-stream PRNG seeds derived from the run's single seed. + +use rand::RngExt; +use rand_xoshiro::Xoshiro256PlusPlus; +use rand_xoshiro::rand_core::SeedableRng; + +/// One PRNG seed per independent stream, all derived from the run's single seed. +/// +/// Every consumer that draws needs a stream nothing else shares. A shared stream +/// couples them: adding a draw anywhere shifts every later draw everywhere, so +/// enabling crash injection would change the network trace, and a bug found at one +/// seed would vanish once an unrelated draw was added. +/// +/// Children are DRAWN from a parent rather than XOR-salted off the seed. Salting +/// needs a distinct constant per stream and silently reuses the parent seed when one +/// is forgotten, which is how the workload and the packet simulator came to share a +/// stream. Drawing needs no constants and cannot hand back a stream without naming +/// it. +/// +/// Pure in `seed`, so every caller derives the same children and nothing has to +/// thread a parent PRNG through the constructors. +#[derive(Debug, Clone, Copy)] +pub struct SimSeeds { + /// Packet delays, drops, replays, partitions and clogs. + pub network: u64, + /// Workload op sampling: which action, which entity, which argument. + pub workload: u64, + /// Task-poll and timer-fire ordering in `DetExecutor`. Separate so shaking out + /// order-dependence never perturbs the network or workload traces. + pub executor: u64, + /// Which shard of a replica receives each inbound packet, modelling the + /// coordinator's connection homing. One draw per delivered packet, the + /// highest-rate stream here. + pub entry_shard: u64, + /// Crash and restart scheduling. Separate so a run with both probabilities at + /// zero draws nothing and replays bit-identically to one with no injector. + pub faults: u64, + /// `PacketSimulatorOptions::swarm`'s parameter draw. Separate from `network` + /// because one seed both picks the network's shape and drives it, and + /// correlating the loss probability with the loss events would collapse what the + /// swarm explores to a diagonal of the parameter space. + pub swarm: u64, +} + +impl SimSeeds { + /// Derive every stream's seed from the run's seed. + /// + /// APPEND fields, never insert. Each draw advances the parent, so a field added + /// in the middle moves every child after it and re-locks every seeded baseline. + #[must_use] + pub fn derive(seed: u64) -> Self { + let mut parent = Xoshiro256PlusPlus::seed_from_u64(seed); + Self { + network: parent.random(), + workload: parent.random(), + executor: parent.random(), + entry_shard: parent.random(), + faults: parent.random(), + swarm: parent.random(), + } + } +} + +#[cfg(test)] +mod tests { + use super::SimSeeds; + use std::collections::HashSet; + + /// Every field its own draw. A copy-paste assigning one child twice would + /// recreate exactly the shared-stream coupling this type exists to prevent, and + /// nothing else in the harness would fail. + #[test] + fn every_stream_gets_its_own_seed() { + let seeds = SimSeeds::derive(0xDEAD_BEEF); + let all = [ + seeds.network, + seeds.workload, + seeds.executor, + seeds.entry_shard, + seeds.faults, + seeds.swarm, + ]; + let unique: HashSet = all.iter().copied().collect(); + assert_eq!(unique.len(), all.len(), "two streams share a seed: {all:?}"); + } + + /// Pure in the seed, which is what lets each constructor derive on its own + /// instead of threading a parent PRNG through every call site. + #[test] + fn derivation_is_a_pure_function_of_the_seed() { + assert_eq!(SimSeeds::derive(7).workload, SimSeeds::derive(7).workload); + assert_ne!(SimSeeds::derive(7).workload, SimSeeds::derive(8).workload); + } +} diff --git a/core/simulator/src/workload/auditor.rs b/core/simulator/src/workload/auditor.rs index d9616ec4ec..ad909d3e6c 100644 --- a/core/simulator/src/workload/auditor.rs +++ b/core/simulator/src/workload/auditor.rs @@ -54,10 +54,27 @@ pub struct AuditorStats { pub replies_unknown: u64, /// Per-action committed counter, indexed by `Action as usize`. pub commits_per_action: [u64; Action::COUNT], - /// Metadata replies carrying a nonzero committed result code (a business - /// rejection). The shadow does not mutate on these; in a serial run the - /// `on_reply` equality oracle asserts the rejection was the targeted outcome. + /// Metadata reply with a nonzero committed result code: a business rejection. + /// Shadow does not mutate; on a serial run `on_reply` asserts it was targeted. pub committed_rejections: u64, + /// Denied before commit: `ReplyHeader::status` set, body EMPTY. The op never + /// entered the log, so the shadow must not move and there is no result section to + /// classify. Shell only; the raw path has no denial site. + pub denials: u64, + /// Per-action denial count and last status, indexed by `Action as usize`. Splits + /// two causes: an op the server refuses for this input (an offset the partition + /// cannot accept yet), versus one dispatch cannot decode at all, a workload bug + /// showing up as every request for that action denied with the same status. + pub denials_per_action: [(u64, u32); Action::COUNT], + /// Result section carrying a transport rejection, not a committed outcome. + /// `build_result_rejection_reply` frames them under the REQUEST's own operation + /// with `status` 0, so only the code distinguishes them from a commit. + pub transient_rejections: u64, + /// Per-action transient count and last code, indexed by `Action as usize`. Split + /// from `denials_per_action` because a denial never passes, while a transient is + /// the cluster mid-view-change or backpressured and clears on retry. An action + /// that is all transients means the workload outruns the cluster, not a broken op. + pub transient_rejections_per_action: [(u64, u32); Action::COUNT], } impl Default for AuditorStats { @@ -67,6 +84,10 @@ impl Default for AuditorStats { replies_unknown: 0, commits_per_action: [0u64; Action::COUNT], committed_rejections: 0, + denials: 0, + denials_per_action: [(0, 0); Action::COUNT], + transient_rejections: 0, + transient_rejections_per_action: [(0, 0); Action::COUNT], } } } @@ -171,6 +192,25 @@ impl ServerAuditor { self.stats.commits_per_action[action as usize] += 1; } + /// Record a pre-commit denial (`ReplyHeader::status` nonzero). + pub const fn note_denial(&mut self, action: Action, status: u32) { + self.stats.denials += 1; + let entry = &mut self.stats.denials_per_action[action as usize]; + entry.0 += 1; + entry.1 = status; + } + + /// Record a result-framed transport rejection (see + /// [`AuditorStats::transient_rejections`]). Neither a commit nor a denial: + /// the shadow does not move, and for `TransientNotCommitted` the request + /// stays outstanding so a replay can settle what actually happened. + pub const fn note_transient_rejection(&mut self, action: Action, code: u32) { + self.stats.transient_rejections += 1; + let entry = &mut self.stats.transient_rejections_per_action[action as usize]; + entry.0 += 1; + entry.1 = code; + } + /// Record a committed business rejection (nonzero result code). Either /// targeted by outcome-first generation (duplicate name, fabricated missing /// entity) or produced by a race. @@ -183,6 +223,26 @@ impl ServerAuditor { &self.stats } + /// Drop every in-flight expectation for `client`, returning how many went. + /// + /// For a client the cluster evicted: its session is gone, so nothing it had + /// outstanding will ever be answered and an expectation left behind would + /// wait forever. The requests themselves were refused before commit, so + /// forgetting them loses no committed state. + pub fn forget_client(&mut self, client: u128) -> usize { + let before = self.in_flight.len(); + self.in_flight.retain(|&(owner, _), _| owner != client); + before - self.in_flight.len() + } + + /// The action of an outstanding request, if one is recorded for `key`. + /// Diagnostic only: names what a stalled run is waiting on, which the bare + /// `(client, request)` pair cannot. + #[must_use] + pub fn in_flight_action(&self, key: (u128, u64)) -> Option { + self.in_flight.get(&key).map(|entry| entry.action) + } + #[must_use] pub fn in_flight_count(&self) -> usize { self.in_flight.len() diff --git a/core/simulator/src/workload/ids.rs b/core/simulator/src/workload/ids.rs index c9422db158..775ef65c4a 100644 --- a/core/simulator/src/workload/ids.rs +++ b/core/simulator/src/workload/ids.rs @@ -22,7 +22,7 @@ //! need pseudo-uuid ids without reshaping this module. use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use rand_xoshiro::rand_core::SeedableRng; #[derive(Debug, Clone, Copy)] @@ -47,7 +47,7 @@ impl IdPermutation { u32::try_from(data).is_ok(), "Random id permutation needs index <= u32::MAX, got {data}" ); - let mut prng = Xoshiro256Plus::seed_from_u64(seed.wrapping_add(data)); + let mut prng = Xoshiro256PlusPlus::seed_from_u64(seed.wrapping_add(data)); let lo32: u32 = prng.random(); ((data & 0xFFFF_FFFF) << 32) | u64::from(lo32) } diff --git a/core/simulator/src/workload/invariants.rs b/core/simulator/src/workload/invariants.rs index bf00d92867..4f6dd3a1d6 100644 --- a/core/simulator/src/workload/invariants.rs +++ b/core/simulator/src/workload/invariants.rs @@ -24,6 +24,7 @@ //! unchanged. use crate::Simulator; +use crate::workload::state_checker::StateChecker; use crate::workload::{CLIENT_REQUEST_QUEUE_MAX, Workload}; use server_common::sharding::IggyNamespace; use std::collections::HashMap; @@ -34,6 +35,9 @@ use std::collections::HashMap; pub struct Invariants { commit_offset: HashMap<(u8, IggyNamespace), u64>, view: HashMap<(u8, IggyNamespace), u64>, + /// Cross-replica committed-log agreement. Runs every tick like the rest, so a + /// divergence is reported where it appears rather than at the next quiesce. + state_checker: StateChecker, } impl Invariants { @@ -49,10 +53,15 @@ impl Invariants { /// - consensus `view` never regresses (a view change only advances it). /// /// Globally: - /// - total in-flight requests stay within the per-client queue ceiling. + /// - total in-flight requests stay within the per-client queue ceiling, + /// - live replicas agree on every committed metadata op they share, and the + /// committed chain stays hash-linked (see [`StateChecker`]). /// - /// Crashed replicas are skipped: their last-seen marks are retained, which - /// stays correct because both quantities are monotonic across a restart. + /// Crashed replicas are skipped and their last-seen marks retained, which + /// holds across a restart for both quantities: the superblock carries `view`, + /// and `Simulator::retain_partition_logs` carries each partition's log so a + /// rebuilt partition recovers its offsets instead of reporting zero. Without + /// that the check trips on a discarded log and calls it a regression. /// /// # Panics /// On any regression or in-flight overflow. The workload seed is in the @@ -89,14 +98,16 @@ impl Invariants { (client_count={}, queue_max={CLIENT_REQUEST_QUEUE_MAX}) (seed={seed:#x})", workload.options.client_count, ); + + self.state_checker.check(sim, seed); } - /// Number of `(replica, namespace)` pairs observed so far. Used by tests to - /// prove the checks ran over live state rather than vacuously. - #[cfg(test)] + /// The canonical committed chain built so far. Tests read it to prove the + /// equality check compared replicas against each other rather than passing + /// over an empty chain. #[must_use] - pub(crate) fn tracked_pairs(&self) -> usize { - self.commit_offset.len() + pub const fn state_checker(&self) -> &StateChecker { + &self.state_checker } } diff --git a/core/simulator/src/workload/mod.rs b/core/simulator/src/workload/mod.rs index 03c7388b9f..0995eb7d7c 100644 --- a/core/simulator/src/workload/mod.rs +++ b/core/simulator/src/workload/mod.rs @@ -33,65 +33,160 @@ pub mod ops; pub mod options; pub mod oracle; pub mod shadow; +pub mod state_checker; use crate::Simulator; use crate::client::SimClient; -use crate::workload::ops::InFlight; +use crate::seeds::SimSeeds; +use crate::workload::ops::{InFlight, InFlightOutcome}; use actions::Action; use auditor::{OnReply, ServerAuditor}; use effect::SimCommand; -use iggy_binary_protocol::{ReplyHeader, RoutedRequestHeader, result_code}; +use iggy_binary_protocol::{Operation, ReplyHeader, RoutedRequestHeader, result_code}; +use iggy_common::IggyError; use invariants::Invariants; use metadata::stm::result::result_code_recognized; use options::WorkloadOptions; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use rand_xoshiro::rand_core::SeedableRng; use server_common::Message; use shadow::Shadow; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; /// Max in-flight requests per client. Must stay under the consensus /// pipeline's queue limits. pub const CLIENT_REQUEST_QUEUE_MAX: usize = 1; +/// An outstanding request, retained so the client can resend it. +/// +/// The encoded message is kept verbatim rather than rebuilt from the sampled +/// `Input`: rebuilding would draw a fresh request id, and a resend must reuse the +/// original, which is what the metadata client table dedups on. A renumbered retry +/// commits a second time instead of returning the cached reply. +struct Outstanding { + message: Message, + /// Replica the most recent attempt went to. A resend moves to the next one, + /// so a client whose primary died eventually finds the new one. + target: u8, + /// Tick of the most recent attempt, not of the first. + attempted_tick: u64, + attempts: u32, +} + +/// One still-unanswered request, as the drain-failure report names it. +/// +/// A struct rather than a five-tuple: every field is a bare integer, so a tuple +/// leaves the reader counting positions in the one report read after a failure. +#[derive(Debug, Clone, Copy)] +pub(crate) struct OutstandingRow { + pub client: u128, + pub request: u64, + pub action: Option, + /// Replica the most recent attempt went to. + pub target: u8, + pub attempts: u32, +} + +/// A transport rejection the dispatch path framed into the result section +/// instead of `ReplyHeader::status`. +/// +/// `build_result_rejection_reply` uses the result section for the `IggyError`s a +/// client must see typed, stamps the REQUEST's own operation, and leaves `status` +/// at 0. Such a reply is shaped exactly like a commit while carrying a code no op's +/// result enum declares, so it has to be recognized before the committed-code path +/// claims it. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub(crate) enum TransientRejection { + /// The request never entered a queue, so it definitely did not commit and is + /// re-issuable anywhere. + NotAccepted, + /// The request may or may not have committed. Only replaying the same request + /// id settles it. + NotCommitted, +} + +impl TransientRejection { + /// `RequestAlreadyApplied` is deliberately absent: the op DID commit and only + /// its reply aged out of the client table's ring, so there is no honest shadow + /// move to make. Left to the recognized-code assert, the right signal for a + /// reply ring too small for the retry latency. + pub(crate) fn from_code(code: u32) -> Option { + if code == IggyError::TransientNotAccepted.as_code() { + Some(Self::NotAccepted) + } else if code == IggyError::TransientNotCommitted.as_code() { + Some(Self::NotCommitted) + } else { + None + } + } +} + pub struct Workload { - prng: Xoshiro256Plus, + prng: Xoshiro256PlusPlus, pub auditor: ServerAuditor, pub shadow: Shadow, pub options: WorkloadOptions, - /// Number of in-flight requests per client. + /// Outstanding requests, keyed as the auditor keys its expectations so the two + /// are removed together. + /// + /// A `BTreeMap`, not a `HashMap`: [`Self::due_resends`] walks it and the submit + /// order is observable, so hash order would make replay diverge from the seed. /// /// TODO: reap on client disconnect; bounded today by the fixed /// `Simulator::new` set. - in_flight_per_client: HashMap, + outstanding: BTreeMap<(u128, u64), Outstanding>, + /// Driver tick, advanced by [`Self::tick`]. A driver that never ticks never + /// resends, which is what the hand-written scenario tests rely on. + now: u64, + /// Total resends issued, for the run summary. + resends: u64, + /// Client evictions survived, for the run summary. Only the dispatch shell + /// produces them, and in practice only once replicas restart under it. + evictions: u64, + /// Clients whose recovery handshake is in flight, and the tick it last went out + /// on. Such a client submits nothing: its session is gone, so anything it sent + /// would be refused and evict it again. + pending_handshakes: BTreeMap, /// Debug counter for `sample()` returning `None` (a targeted outcome whose /// shadow precondition is unmet). Flags PRNG-trace drift during development. samples_none: u64, - /// Assert the targeted outcome equals the committed one. Sound only for a - /// fully serial run (one client, one in-flight slot), where the shadow equals - /// committed server state at sample time so the target is always realized. - /// Gated on `client_count == 1 && CLIENT_REQUEST_QUEUE_MAX == 1`. + /// Whether the run is fully serial (one client, one in-flight slot), the + /// standing precondition for comparing the shadow against committed state. + /// Fixed at construction, unlike `strict_outcome_oracle`, the arm/disarm bit on + /// top: an eviction disarms the oracle without making the run concurrent, and + /// the quiesce comparison still runs to decide whether it can be re-armed. + serial_run: bool, + /// Assert the targeted outcome equals the committed one. Sound only on a serial + /// run, where the shadow equals committed state at sample time so the target is + /// always realized. Starts at `serial_run`, disarmed by + /// [`Self::forget_evicted_client`], re-armed by [`Self::rearm_outcome_oracle`]. strict_outcome_oracle: bool, } impl Workload { #[must_use] pub fn new(options: WorkloadOptions) -> Self { - let prng = Xoshiro256Plus::seed_from_u64(options.seed); + let prng = Xoshiro256PlusPlus::seed_from_u64(SimSeeds::derive(options.seed).workload); let shadow = Shadow::new(options.namespaces.clone(), ids::IdPermutation::Identity); // Both halves of the soundness precondition (see the field doc). Coupling // to the queue max disarms strict equality if it is raised, rather than // letting the assert fire on a legitimately raced outcome (a 2nd in-flight // request sampled against the shadow before the 1st commits). - let strict_outcome_oracle = options.client_count == 1 && CLIENT_REQUEST_QUEUE_MAX == 1; + let serial_run = options.client_count == 1 && CLIENT_REQUEST_QUEUE_MAX == 1; + let strict_outcome_oracle = serial_run; Self { prng, auditor: ServerAuditor::new(), shadow, options, - in_flight_per_client: HashMap::new(), + outstanding: BTreeMap::new(), + now: 0, + resends: 0, + evictions: 0, + pending_handshakes: BTreeMap::new(), samples_none: 0, + serial_run, strict_outcome_oracle, } } @@ -99,18 +194,172 @@ impl Workload { /// True if the client has a free in-flight slot. #[must_use] pub fn client_idle(&self, client_id: u128) -> bool { - self.in_flight_per_client - .get(&client_id) - .copied() - .unwrap_or(0) - < CLIENT_REQUEST_QUEUE_MAX + self.client_in_flight(client_id) < CLIENT_REQUEST_QUEUE_MAX } /// Total in-flight requests across all clients. Read by the /// [`Invariants`]; draws no PRNG. #[must_use] pub(crate) fn total_in_flight(&self) -> usize { - self.in_flight_per_client.values().copied().sum() + self.outstanding.len() + } + + /// Advance the resend clock by one tick. Called once per driver iteration; + /// [`Self::due_resends`] measures against it. + pub const fn tick(&mut self) { + self.now += 1; + } + + /// Total resends issued so far. + #[must_use] + pub const fn resends(&self) -> u64 { + self.resends + } + + /// Total client evictions survived. + #[must_use] + pub const fn evictions(&self) -> u64 { + self.evictions + } + + /// Forget everything outstanding for a client the cluster evicted. + /// + /// An eviction is session-terminal: the server refused the request BEFORE commit + /// (`Eviction(NoSession)` from an unbound transport, which a replica restart + /// leaves behind, session bindings living in the per-connection + /// `SessionManager`). Resending is not an option, the retained message carrying + /// the old session id; the client logs in again and samples afresh. + /// + /// The forgotten request's fate is genuinely unknown, which is why this disarms + /// the strict outcome oracle. The refusal proves only that the ATTEMPT drawing + /// it did not commit, and that attempt may have been a resend of a request whose + /// original committed with its reply lost. The shadow is then missing an effect + /// that did happen, and every later targeted outcome can disagree with what + /// commits. Claiming the shadow is still authoritative would turn a known + /// unknown into a spurious failure. + /// + /// Returns how many requests were forgotten. + pub fn forget_evicted_client(&mut self, client_id: u128) -> usize { + self.evictions += 1; + self.strict_outcome_oracle = false; + self.outstanding.retain(|&(owner, _), _| owner != client_id); + self.auditor.forget_client(client_id) + } + + /// Whether this client is waiting on a recovery handshake and so must not be + /// asked for a request. + #[must_use] + pub fn is_recovering(&self, client_id: u128) -> bool { + self.pending_handshakes.contains_key(&client_id) + } + + /// Record that a recovery handshake for `client_id` went out to `target`. + pub fn note_handshake_submitted(&mut self, client_id: u128, target: u8) { + self.pending_handshakes + .insert(client_id, (target, self.now)); + } + + /// Handshakes whose reply is overdue, each paired with the replica to retry + /// against. Rotates targets for the same reason `due_resends` does: the replica + /// that looked live may be neither reachable nor able to forward. + /// + /// The interval is the request timeout rather than something tighter because + /// every landed register commits and re-fences (see `register_preflight`), so an + /// eager retry costs the client an extra recovery round trip. + #[must_use = "returned handshakes must be submitted or the client stays wedged"] + pub fn due_handshake_resends(&mut self, replica_count: u8) -> Vec<(u128, u8)> { + let timeout = self.options.request_timeout_ticks; + if timeout == 0 { + return Vec::new(); + } + let now = self.now; + let replica_count = replica_count.max(1); + let mut due = Vec::new(); + for (&client_id, (target, submitted)) in &mut self.pending_handshakes { + if now.saturating_sub(*submitted) < timeout { + continue; + } + *target = (*target + 1) % replica_count; + *submitted = now; + due.push((client_id, *target)); + } + due + } + + /// Take the pending handshake for the client this reply answers, if it is one. + /// + /// Returns the client id so the caller can bind the session; `None` leaves the + /// reply for [`Self::on_reply`], which is where every non-handshake reply + /// belongs. + pub fn take_pending_handshake(&mut self, reply: &Message) -> Option { + let header = reply.header(); + if header.operation != Operation::Register { + return None; + } + self.pending_handshakes + .remove(&header.client) + .map(|_| header.client) + } + + /// Requests whose reply has not arrived within + /// [`WorkloadOptions::request_timeout_ticks`], each paired with the replica + /// to retry it against. Callers must submit every returned message. + /// + /// What a real client's read timeout does, needed for two reasons. A dropped + /// request or reply otherwise strands the client's only in-flight slot for the + /// rest of the run, so any packet loss wedges the workload. And a request lost + /// to a crashed primary can only be answered by the next one, which the client + /// reaches by rotating its target. + /// + /// Safe on both planes but not equally cheap: the metadata plane dedups on the + /// retained request id and replays the cached reply, while the partition plane + /// is at-least-once and may commit twice. The shadow models that, `Effect` + /// application being driven by what committed rather than what was targeted. + #[must_use = "returned requests must be submitted or the client stays wedged"] + pub fn due_resends(&mut self) -> Vec<(u8, Message)> { + let timeout = self.options.request_timeout_ticks; + if timeout == 0 { + return Vec::new(); + } + let replica_count = self.options.replica_count.max(1); + let now = self.now; + let mut due = Vec::new(); + for entry in self.outstanding.values_mut() { + if now.saturating_sub(entry.attempted_tick) < timeout { + continue; + } + entry.target = (entry.target + 1) % replica_count; + entry.attempted_tick = now; + entry.attempts += 1; + due.push((entry.target, entry.message.deep_copy())); + } + self.resends += due.len() as u64; + due + } + + /// Outstanding requests in key order. Diagnostic only: names what a run was + /// waiting on when it failed to drain, including which op, since some cannot be + /// answered twice and a resend of those stalls permanently. + #[must_use] + pub(crate) fn outstanding_summary(&self) -> Vec { + self.outstanding + .iter() + .map(|(&key, entry)| OutstandingRow { + client: key.0, + request: key.1, + action: self.auditor.in_flight_action(key), + target: entry.target, + attempts: entry.attempts, + }) + .collect() + } + + /// In-flight count for one client. Keys are `(client, request)`, so the + /// client's entries are one contiguous range. + fn client_in_flight(&self, client_id: u128) -> usize { + self.outstanding + .range((client_id, 0)..=(client_id, u64::MAX)) + .count() } /// Aggregate in-flight ceiling: one queue's worth per declared client. @@ -121,15 +370,40 @@ impl Workload { usize::from(self.options.client_count) * CLIENT_REQUEST_QUEUE_MAX } - /// True when the run is fully serial (one client, one in-flight slot), the - /// regime where the shadow equals committed server state. Gates the - /// quiesce-time entity oracle the same way it gates the per-op equality - /// oracle. + /// Whether the entity oracle is currently armed. A driver reports it: a run + /// whose oracle was disarmed by an eviction and never re-armed asserted nothing + /// about entity state, and without this looks identical to one that did. #[must_use] - pub(crate) const fn strict_outcome_oracle(&self) -> bool { + pub const fn strict_outcome_oracle(&self) -> bool { self.strict_outcome_oracle } + /// True when the run is fully serial (one client, one in-flight slot), the + /// regime where the shadow equals committed state. Separate from the arm/disarm + /// bit above because an eviction disarms the oracle without making the run + /// concurrent. + #[must_use] + pub const fn serial_run(&self) -> bool { + self.serial_run + } + + /// Re-arm the outcome oracle after the shadow has been PROVEN to equal + /// committed state again. + /// + /// [`Self::forget_evicted_client`] disarms because a forgotten request's fate is + /// unknown, and an unknown can leave the shadow missing an effect that happened. + /// But an unknown is not proof of divergence: once the shadow and the leader's + /// committed metadata are observed equal at rest, it has resolved in the + /// shadow's favour. + /// + /// Without this, one eviction turns the entity oracle into a no-op for the rest + /// of the run, and a fault run evicts routinely, so the strongest oracle was + /// silently off for almost every run that mattered. Callable only from + /// [`oracle::assert_converged`], which does the comparison this rests on. + pub(crate) const fn rearm_outcome_oracle(&mut self) { + self.strict_outcome_oracle = true; + } + /// Build the next request for `client`. Returns the message and target /// replica index, or `None` if the client has no idle slot or /// `ops::sample` could not synthesize an input. @@ -173,10 +447,15 @@ impl Workload { request_namespace: header.group, }, ); - *self - .in_flight_per_client - .entry(client.client_id()) - .or_insert(0) += 1; + self.outstanding.insert( + key, + Outstanding { + message: message.deep_copy(), + target, + attempted_tick: self.now, + attempts: 1, + }, + ); Some((target, message)) } @@ -200,12 +479,28 @@ impl Workload { OnReply::Match(entry) => entry, OnReply::NsMismatch => { // Entry consumed; release slot, skip effects (misrouted). - self.decrement_in_flight(header.client); + self.release_outstanding(key); return Vec::new(); } OnReply::Unknown => return Vec::new(), }; + // A pre-commit denial short-circuits everything below. The two channels are + // mutually exclusive: a reply either commits (status 0, result section + // present) or is denied before commit (status set, EMPTY body). Reading a + // result section off a denial finds no bytes, which the metadata branch + // below would report as a corrupt reply. + // + // The op never entered the log, so the shadow must not move and there is + // nothing to classify. Only the dispatch shell produces these, since + // authorization runs there, which is why this went unmodelled until the + // workload ran through the shell. + if header.status != 0 { + self.auditor.note_denial(entry.action, header.status); + self.release_outstanding(key); + return Vec::new(); + } + // Decode the committed result code. Metadata replies carry a // result section (see `ApplyReply::to_reply_body`); partition-plane // replies do not, hence the `is_metadata` gate. @@ -234,6 +529,30 @@ impl Workload { entry.action, header.client, header.request, ); }; + // A transport rejection, not a committed outcome. Dispatch refuses a + // request it cannot place (not the primary, transferring, request queue + // full, a view change canceled the pending prepare) and answers with + // the reason in the result section under the request's own operation. + // Nothing committed, so the committed-code path below must not claim it, + // and the assert after that would report it as a server bug. + if let Some(transient) = TransientRejection::from_code(code) { + self.auditor.note_transient_rejection(entry.action, code); + match transient { + // Never entered a queue, so the shadow must not move and the + // client's slot is free for a fresh sample. + TransientRejection::NotAccepted => self.release_outstanding(key), + // Outcome unknown. Releasing would leave the shadow guessing + // whether the op landed, so hold the request outstanding and let + // `due_resends` replay the same request id: the primary answers + // from its client table if it committed and re-dispatches if it + // did not, which turns the unknown into a fact. + TransientRejection::NotCommitted => { + self.auditor.record_in_flight(key, entry); + } + } + return Vec::new(); + } + // The state machine only commits codes its own result enum declares, // so an unrecognized one is a server bug (a race still yields a // declared code). Classify never guesses. @@ -253,6 +572,29 @@ impl Workload { // Classify the *actual* committed outcome from the wire result code. let classified = ops::classify_reply(entry.action, committed_code); + // `CreatePersonalAccessToken` is the one op whose REPLAY is refused rather + // than served from the reply cache: the committed secret is unrecoverable, + // so a re-minted one would not match the stored hash and the metadata plane + // answers `PersonalAccessTokenAlreadyExists`. On a resend that refusal proves + // the ORIGINAL attempt committed, so the shadow has to record the token it + // added or every later name draw is made against a shadow missing one. A + // FIRST attempt answering the same code is a genuine duplicate, which is why + // the attempt count is the discriminator. + let attempts = self.outstanding.get(&key).map_or(1, |entry| entry.attempts); + let classified = if attempts > 1 + && entry.outcome + == InFlightOutcome::CreatePersonalAccessToken( + ops::create_personal_access_token::Outcome::Ok, + ) + && classified + == InFlightOutcome::CreatePersonalAccessToken( + ops::create_personal_access_token::Outcome::AlreadyExists, + ) { + entry.outcome + } else { + classified + }; + // Equality oracle: the targeted outcome must match what committed. Sound // only for a fully serial run (see `strict_outcome_oracle`); with several // clients a concurrent commit can flip it (a targeted duplicate races a @@ -283,24 +625,26 @@ impl Workload { self.auditor.note_committed(entry.action); } - self.decrement_in_flight(header.client); + self.release_outstanding(key); result.sim_commands } - /// Release one in-flight slot. Panics on underflow so a future - /// double-decrement surfaces instead of being silently clamped. + /// Drop a request's retry entry, freeing the client's slot. Paired with the + /// auditor consuming its expectation for the same key. /// /// # Panics - /// Panics if no entry exists for `client`, or if the counter is 0. - fn decrement_in_flight(&mut self, client: u128) { - let count = self - .in_flight_per_client - .get_mut(&client) - .expect("decrement_in_flight: no entry for client; record_in_flight must precede"); - *count = count - .checked_sub(1) - .expect("in_flight underflow: per-client counter went below 0"); + /// If no entry exists for `key`. The auditor reports a match or a namespace + /// mismatch only for a key it was given, and `build_request` records both sides + /// together, so a miss means the two drifted. + fn release_outstanding(&mut self, key: (u128, u64)) { + assert!( + self.outstanding.remove(&key).is_some(), + "no outstanding entry for (client={}, request={}); the auditor \ + matched a key the retry buffer never recorded", + key.0, + key.1, + ); } /// Debug counter for `sample()` returning `None`. Surfaces sampling @@ -351,18 +695,14 @@ impl Workload { } } -/// Salt mixed into the workload seed for the fault PRNG, so crash scheduling -/// is reproducible from the seed yet independent of the traffic draw order -/// (the determinism baseline stays valid with injection on). -const FAULT_SEED_SALT: u64 = 0x5A1A_F0E5_FACE_0001; - /// Drive the simulator until `tick_budget` elapses or `replies_target` /// replies are seen. Returns the number of replies seen. /// -/// The invariants are asserted after every tick, so a consensus or -/// workload regression panics at the tick it occurs (the seed in the message -/// replays it). When `crash_per_tick_ratio > 0` the driver also injects -/// crash-only faults via `maybe_inject_crash`. +/// The invariants are asserted after every tick, so a consensus or workload +/// regression panics at the tick it occurs (the seed in the message replays it). +/// Crash and restart injection runs through [`FaultInjector`], idle unless one of +/// the two probabilities is set. Discards the injector; use [`run_with_faults`] to +/// read the crash and restart counts back. pub fn run( sim: &mut Simulator, workload: &mut Workload, @@ -370,23 +710,72 @@ pub fn run( tick_budget: u64, replies_target: u64, ) -> u64 { + let mut injector = FaultInjector::new(workload.options.seed, sim.replica_count); + run_with_faults( + sim, + workload, + clients, + tick_budget, + replies_target, + &mut injector, + ) +} + +/// [`run`] against a caller-owned [`FaultInjector`], so a test can assert what +/// was actually injected instead of trusting the probabilities to have fired. +/// # Panics +/// If `injector` was built for a different replica count than `sim` has. +pub fn run_with_faults( + sim: &mut Simulator, + workload: &mut Workload, + clients: &[SimClient], + tick_budget: u64, + replies_target: u64, + injector: &mut FaultInjector, +) -> u64 { + // The injector is caller-owned, and it sized `last_transition` from a count + // nobody has checked against this simulator. Left unchecked the mismatch + // surfaces as a bare "index out of bounds" from inside `stable_for`, with no + // seed, replica or injector named, in a harness whose every panic is triaged as + // "real bug or artifact?". + assert_eq!( + injector.replica_count(), + sim.replica_count, + "fault injector was built for {} replicas but this simulator has {} \ + (seed={:#x})", + injector.replica_count(), + sim.replica_count, + workload.options.seed, + ); let mut invariants = Invariants::new(); - let mut fault_prng = Xoshiro256Plus::seed_from_u64(workload.options.seed ^ FAULT_SEED_SALT); let mut replies_seen = 0u64; for _ in 0..tick_budget { - if workload.options.crash_per_tick_ratio > 0.0 { - maybe_inject_crash(sim, workload, &mut fault_prng); - } + workload.tick(); + injector.step(sim, workload); + // Resend before sampling: a timed-out request still holds the client's + // slot, so `build_request` would decline it anyway. + resubmit_due(sim, workload); + resubmit_due_handshakes(sim, workload, clients); for client in clients { + // A client whose session is gone submits nothing: the request would be + // refused and evict it again, and the eviction would look like a fresh + // failure rather than the one already being recovered. + if workload.is_recovering(client.client_id()) { + continue; + } if let Some((target, msg)) = workload.build_request(client) { sim.submit_request(client.client_id(), target, msg.into_generic()); } } for reply in sim.step() { + if bind_recovered_client(sim, workload, clients, &reply) { + continue; + } let cmds = workload.on_reply(&reply); apply_sim_commands(sim, &cmds); replies_seen += 1; } + recover_evicted_clients(sim, workload, clients); invariants.check(sim, workload); if replies_seen >= replies_target { break; @@ -395,48 +784,247 @@ pub fn run( replies_seen } -/// With probability `crash_per_tick_ratio`, crash one live non-primary replica, -/// provided doing so leaves at least `min_survivors` live. Crash-only: a -/// crashed replica is never restarted (that needs consensus durability). +/// Crash and restart injection with stability windows: a crash must last a while +/// before it may be repaired, and a repaired replica must run a while before it may +/// fail again. /// -/// "Non-primary" is partition-plane only: the exclusion set comes from -/// `Simulator::primary_index`, which reads `partitions()`. The metadata-plane -/// primary is not consulted; it is spared only by co-location, since every group -/// starts at view 0 with `primary = view % replica_count` (so replica 0 leads -/// both planes) and `min_survivors` keeps a commit quorum, so no view change -/// moves it. Were the two planes' primaries to diverge, the metadata primary -/// could be crashed. +/// Owns the fault PRNG so crash scheduling stays reproducible from the seed yet +/// independent of the traffic draw order. Draws nothing while both probabilities +/// are zero, so a fault-free run replays bit-identically. +pub struct FaultInjector { + prng: Xoshiro256PlusPlus, + /// Tick of each replica's last crash or restart, indexed by replica id. + /// Compared against the stability windows to decide eligibility. + last_transition: Vec, + now: u64, + crashes: u64, + restarts: u64, +} + +impl FaultInjector { + #[must_use] + pub fn new(seed: u64, replica_count: u8) -> Self { + Self { + prng: Xoshiro256PlusPlus::seed_from_u64(SimSeeds::derive(seed).faults), + last_transition: vec![0; usize::from(replica_count)], + now: 0, + crashes: 0, + restarts: 0, + } + } + + /// Replicas this injector was built for. A driver checks it against the + /// simulator it is about to drive; see the assert in [`run_with_faults`]. + #[must_use] + pub fn replica_count(&self) -> u8 { + u8::try_from(self.last_transition.len()).unwrap_or(u8::MAX) + } + + #[must_use] + pub const fn crashes(&self) -> u64 { + self.crashes + } + + #[must_use] + pub const fn restarts(&self) -> u64 { + self.restarts + } + + /// Advance one tick and maybe crash or restart one replica. + /// + /// Restart is considered before crash so a single tick never both revives + /// and kills, which would make the stability windows meaningless. + pub fn step(&mut self, sim: &mut Simulator, workload: &Workload) { + self.now += 1; + self.maybe_restart(sim, workload); + self.maybe_crash(sim, workload); + } + + /// With probability `restart_per_tick_ratio`, restart one replica that has + /// been down at least `crash_stability_ticks`. + /// + /// What exercises rejoin: the replica comes back with its durable superblock and + /// metadata WAL but no volatile consensus state, asks the current view's primary + /// for a `StartView`, and repairs the log it missed. + fn maybe_restart(&mut self, sim: &mut Simulator, workload: &Workload) { + if workload.options.restart_per_tick_ratio <= 0.0 { + return; + } + let eligible: Vec = (0..sim.replica_count) + .filter(|replica_idx| sim.is_crashed(*replica_idx)) + .filter(|replica_idx| { + self.stable_for(*replica_idx) >= workload.options.crash_stability_ticks + }) + .collect(); + if eligible.is_empty() { + return; + } + let roll: f32 = self.prng.random(); + if roll >= workload.options.restart_per_tick_ratio { + return; + } + let revived = eligible[self.prng.random_range(0..eligible.len())]; + sim.replica_restart(revived); + self.last_transition[usize::from(revived)] = self.now; + self.restarts += 1; + } + + /// With probability `crash_per_tick_ratio`, crash one live replica that has + /// been up at least `restart_stability_ticks`, provided doing so leaves at + /// least `min_survivors` live. + /// + /// Primaries are excluded unless `spare_primary` is off. The exclusion set comes + /// from `Simulator::primary_index`, which reads `partitions()`, so it names + /// partition-plane primaries; the metadata primary is spared only by co-location, + /// every group starting at view 0 with `primary = view % replica_count`. Once + /// views diverge across planes it can be crashed even with this on. + fn maybe_crash(&mut self, sim: &mut Simulator, workload: &Workload) { + if workload.options.crash_per_tick_ratio <= 0.0 { + return; + } + let live: Vec = (0..sim.replica_count) + .filter(|replica_idx| !sim.is_crashed(*replica_idx)) + .collect(); + if live.len() <= usize::from(workload.options.min_survivors) { + return; + } + let roll: f32 = self.prng.random(); + if roll >= workload.options.crash_per_tick_ratio { + return; + } + let primaries: HashSet = if workload.options.spare_primary { + workload + .options + .namespaces + .iter() + .filter_map(|ns| sim.primary_index(*ns)) + .collect() + } else { + HashSet::new() + }; + let eligible: Vec = live + .into_iter() + .filter(|replica_idx| !primaries.contains(replica_idx)) + .filter(|replica_idx| { + self.stable_for(*replica_idx) >= workload.options.restart_stability_ticks + }) + .collect(); + if eligible.is_empty() { + return; + } + let victim = eligible[self.prng.random_range(0..eligible.len())]; + sim.replica_crash(victim); + self.last_transition[usize::from(victim)] = self.now; + self.crashes += 1; + } + + /// Ticks since this replica last changed state. A replica that never + /// transitioned counts from tick 0, so the first crash still has to wait out + /// `restart_stability_ticks`. + fn stable_for(&self, replica_idx: u8) -> u64 { + self.now + .saturating_sub(self.last_transition[usize::from(replica_idx)]) + } +} + +/// Log any evicted client back in, which is what a real client does. /// -/// Primaries are spared at all because the driver has no request-timeout/resend -/// path: a request lost to a crashed primary would wedge the client's only -/// in-flight slot. Forcing primary crashes (and the view change they trigger) -/// while keeping traffic flowing is future work gated on that resend path. -fn maybe_inject_crash(sim: &mut Simulator, workload: &Workload, prng: &mut Xoshiro256Plus) { - let live: Vec = (0..sim.replica_count) - .filter(|replica_idx| !sim.is_crashed(*replica_idx)) - .collect(); - if live.len() <= usize::from(workload.options.min_survivors) { - return; - } - let roll: f32 = prng.random(); - if roll >= workload.options.crash_per_tick_ratio { - return; - } - let primaries: HashSet = workload - .options - .namespaces +/// A replica restart drops its `SessionManager`, bindings being per-connection and +/// volatile, so a client that had a session there is unbound and its next +/// replicated request is refused with `Eviction(NoSession)`. The client table is +/// replicated metadata and survives, so the re-login rebinds the existing entry +/// (bumping its fence epoch) and request numbering continues. +/// +/// Outstanding requests are forgotten rather than resent: the retained message +/// carries the old session id, so it would only be refused again. See +/// [`Workload::forget_evicted_client`]. +/// +/// # Panics +/// If an evicted client id is not one the driver knows about, which would mean the +/// simulator and the driver disagree about who is connected. +fn recover_evicted_clients(sim: &mut Simulator, workload: &mut Workload, clients: &[SimClient]) { + for client_id in sim.take_evictions() { + let client = clients + .iter() + .find(|client| client.client_id() == client_id) + .unwrap_or_else(|| { + panic!("cluster evicted unknown client {client_id}: not one the driver drives") + }); + workload.forget_evicted_client(client_id); + // Any live replica: a client dialing a backup is a supported path (the + // backup forwards the register), and the default target may itself be the + // replica whose restart caused the eviction, in which case the login just + // times out. + let Some(target) = (0..sim.replica_count).find(|idx| !sim.is_crashed(*idx)) else { + continue; + }; + // Submitted, not awaited. The blocking helpers step the simulator up to + // `SETUP_TOTAL_STEPS` times inside this tick, with no `Workload::tick`, no + // fault injection and no invariant check, which is the per-tick checking + // `run_with_faults` exists for. The reply is picked up by the driver's loop. + // + // `submit_handshake` also picks the frame this simulator's mode can answer. + // Evictions are not shell-only: `Simulator::step` records them without + // consulting the mode, and the raw wire ingress sends them for `NoSession` + // and `SessionTooLow`. A login on the raw path then fails quietly rather + // than loudly, `SimClient::login` being an `Operation::Register` frame that + // the raw ingress answers by registering, so the client walks away with a + // session minted by a path that never read the credentials while + // `set_shell_wire` ends its coverage of the replicated PAT path. + sim.submit_handshake(client, target); + workload.note_handshake_submitted(client_id, target); + } +} + +/// Re-submit any recovery handshake whose reply is overdue. +fn resubmit_due_handshakes(sim: &mut Simulator, workload: &mut Workload, clients: &[SimClient]) { + for (client_id, target) in workload.due_handshake_resends(sim.replica_count) { + if let Some(client) = clients + .iter() + .find(|client| client.client_id() == client_id) + { + sim.submit_handshake(client, target); + } + } +} + +/// Bind the session a recovery handshake reply carries, returning whether the reply +/// was one. A transient rejection is not an answer, so the handshake stays pending +/// and [`Workload::due_handshake_resends`] retries it. +fn bind_recovered_client( + sim: &Simulator, + workload: &mut Workload, + clients: &[SimClient], + reply: &Message, +) -> bool { + let Some(session) = sim.handshake_session(reply) else { + // Still a handshake reply if it correlates, just not a usable one: leave the + // entry pending so the retry fires, and swallow it either way, the auditor + // having no expectation for a handshake. + return reply.header().operation == Operation::Register + && workload.is_recovering(reply.header().client); + }; + let Some(client_id) = workload.take_pending_handshake(reply) else { + return false; + }; + if let Some(client) = clients .iter() - .filter_map(|ns| sim.primary_index(*ns)) - .collect(); - let eligible: Vec = live - .into_iter() - .filter(|replica_idx| !primaries.contains(replica_idx)) - .collect(); - if eligible.is_empty() { - return; - } - let victim = eligible[prng.random_range(0..eligible.len())]; - sim.replica_crash(victim); + .find(|client| client.client_id() == client_id) + { + client.bind_session(session); + } + true +} + +/// Submit every request whose reply is overdue (see [`Workload::due_resends`]). +/// +/// The client id rides the retained message's header, so a resend re-enters the +/// network exactly as the original did, only aimed at the next replica. +pub fn resubmit_due(sim: &mut Simulator, workload: &mut Workload) { + for (target, message) in workload.due_resends() { + let client_id = message.header().client; + sim.submit_request(client_id, target, message.into_generic()); + } } /// Apply `SimCommand`s returned by [`Workload::on_reply`]. @@ -451,3 +1039,91 @@ pub fn apply_sim_commands(sim: &mut Simulator, cmds: &[SimCommand]) { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::packet::PacketSimulatorOptions; + use server_common::sharding::IggyNamespace; + + /// A raw-path eviction is recovered by re-registering, not by a shell login. + /// + /// Evictions are not shell-only. `Simulator::step` classifies an `Eviction` frame + /// without consulting the mode, and the raw wire ingress produces them: + /// `IggyMetadata::on_request` runs `request_preflight` through + /// `apply_preflight_consensus_plane`, which sends one for `NoSession` and + /// `SessionTooLow`. + /// + /// The wrong branch fails silently, hence asserting on the wire shape rather + /// than on a panic: the raw ingress answers `SimClient::login` as a plain + /// register, so the client ends up with a session minted by a path that never + /// read the credentials, while `shell_login_via`'s `set_shell_wire` ends its + /// coverage of the replicated PAT path for good. + #[test] + fn a_raw_eviction_is_recovered_by_re_registering() { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + + let replica_count: u8 = 3; + let client_id: u128 = 1; + let seed = 0xE71C_7100; + let network_opts = PacketSimulatorOptions { + node_count: replica_count, + client_count: 1, + seed, + ..PacketSimulatorOptions::default() + }; + let mut sim = Simulator::new( + usize::from(replica_count), + std::iter::once(client_id), + network_opts, + ); + assert!( + !sim.is_shell(), + "this test covers the raw path; a shell simulator would take the login branch" + ); + let ns = IggyNamespace::new(1, 1, 0); + sim.init_partition(ns); + + let client = SimClient::new(client_id); + sim.register_client_with_primary(&client); + + let options = WorkloadOptions::new(seed, replica_count, vec![ns]); + let mut workload = Workload::new(options); + let clients = [client]; + + // Stands in for the frame the wire ingress sends: `step` records the id + // the same way whatever produced it, so the driver sees exactly this. + sim.evicted.push(client_id); + recover_evicted_clients(&mut sim, &mut workload, &clients); + + assert!( + !clients[0].shell_wire(), + "a raw client was flipped to the client wire shape, so its PAT requests \ + stop exercising the replicated path for the rest of the run" + ); + + // The recovery has to leave a usable session behind. Without a fresh + // registration the next request is refused with another eviction and + // nothing commits. + let replies = run(&mut sim, &mut workload, &clients, 400, u64::MAX); + assert!(replies > 0, "the recovered client got no replies"); + assert!( + workload + .auditor + .stats() + .commits_per_action + .iter() + .sum::() + > 0, + "the recovered client committed nothing, so its session was not restored" + ); + assert!( + sim.take_evictions().is_empty(), + "the recovered client was evicted again, so the re-registration did not bind" + ); + } +} diff --git a/core/simulator/src/workload/ops/change_password.rs b/core/simulator/src/workload/ops/change_password.rs index 59668c92c9..ecec625acd 100644 --- a/core/simulator/src/workload/ops/change_password.rs +++ b/core/simulator/src/workload/ops/change_password.rs @@ -20,7 +20,7 @@ use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -42,7 +42,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::UserNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/create_consumer_group.rs b/core/simulator/src/workload/ops/create_consumer_group.rs index df1bc5c639..342a44b35b 100644 --- a/core/simulator/src/workload/ops/create_consumer_group.rs +++ b/core/simulator/src/workload/ops/create_consumer_group.rs @@ -22,7 +22,7 @@ //! fabricated topic), or `NameAlreadyExists` (an existing group name). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -49,7 +49,7 @@ pub const OUTCOMES: &[Outcome] = &[ pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/create_partitions.rs b/core/simulator/src/workload/ops/create_partitions.rs index 2ad2d3ed24..0b620a2c30 100644 --- a/core/simulator/src/workload/ops/create_partitions.rs +++ b/core/simulator/src/workload/ops/create_partitions.rs @@ -26,7 +26,7 @@ use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -48,7 +48,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/create_personal_access_token.rs b/core/simulator/src/workload/ops/create_personal_access_token.rs index 561b515343..f392ed175f 100644 --- a/core/simulator/src/workload/ops/create_personal_access_token.rs +++ b/core/simulator/src/workload/ops/create_personal_access_token.rs @@ -20,7 +20,7 @@ //! (expiry = 0). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -41,7 +41,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::AlreadyExists]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { let name = match outcome { diff --git a/core/simulator/src/workload/ops/create_stream.rs b/core/simulator/src/workload/ops/create_stream.rs index 390069c36b..89b43b9ee1 100644 --- a/core/simulator/src/workload/ops/create_stream.rs +++ b/core/simulator/src/workload/ops/create_stream.rs @@ -19,7 +19,7 @@ //! by reusing a live stream name from the shadow. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -39,7 +39,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::NameAlreadyExists]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/create_topic.rs b/core/simulator/src/workload/ops/create_topic.rs index 3e9a12916f..b097a53939 100644 --- a/core/simulator/src/workload/ops/create_topic.rs +++ b/core/simulator/src/workload/ops/create_topic.rs @@ -21,7 +21,7 @@ use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -47,7 +47,7 @@ pub const OUTCOMES: &[Outcome] = &[ pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/create_user.rs b/core/simulator/src/workload/ops/create_user.rs index c002d9bbc3..037ba44085 100644 --- a/core/simulator/src/workload/ops/create_user.rs +++ b/core/simulator/src/workload/ops/create_user.rs @@ -19,7 +19,7 @@ //! live username from the shadow). Status fixed at 1 (Active). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -41,7 +41,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::UserAlreadyExists]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { let username = match outcome { diff --git a/core/simulator/src/workload/ops/delete_consumer_group.rs b/core/simulator/src/workload/ops/delete_consumer_group.rs index 996ea1d3c1..2270ba1bba 100644 --- a/core/simulator/src/workload/ops/delete_consumer_group.rs +++ b/core/simulator/src/workload/ops/delete_consumer_group.rs @@ -23,7 +23,7 @@ //! name), mirroring the legacy resolution ladder. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -50,7 +50,7 @@ pub const OUTCOMES: &[Outcome] = &[ pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/delete_consumer_offset.rs b/core/simulator/src/workload/ops/delete_consumer_offset.rs index 45e61ae01b..7bffb95345 100644 --- a/core/simulator/src/workload/ops/delete_consumer_offset.rs +++ b/core/simulator/src/workload/ops/delete_consumer_offset.rs @@ -19,12 +19,13 @@ use iggy_binary_protocol::{AckLevel, RoutedRequestHeader}; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use server_common::sharding::IggyNamespace; use crate::client::SimClient; use crate::workload::effect::Effect; +use crate::workload::ops::sample_consumer_kind; use crate::workload::options::WorkloadOptions; use crate::workload::shadow::Shadow; @@ -46,13 +47,13 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Success]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, options: &WorkloadOptions, ) -> Option { match outcome { Outcome::Success => { let ns = shadow.pick_namespace(prng)?; - let consumer_kind: u8 = u8::from(prng.random::()); + let consumer_kind = sample_consumer_kind(prng); let consumer_id: u32 = prng.random_range(0..options.consumer_pool_size.max(1)); let f: f32 = prng.random(); let ack = if f < options.ack_quorum_ratio { diff --git a/core/simulator/src/workload/ops/delete_partitions.rs b/core/simulator/src/workload/ops/delete_partitions.rs index 3711f89ddf..a5994b5ac0 100644 --- a/core/simulator/src/workload/ops/delete_partitions.rs +++ b/core/simulator/src/workload/ops/delete_partitions.rs @@ -26,7 +26,7 @@ use iggy_binary_protocol::{MAX_PARTITIONS_PER_REQUEST, RoutedRequestHeader}; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -53,7 +53,7 @@ pub const OUTCOMES: &[Outcome] = &[ pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/delete_personal_access_token.rs b/core/simulator/src/workload/ops/delete_personal_access_token.rs index a829f5fa73..34d11b41f7 100644 --- a/core/simulator/src/workload/ops/delete_personal_access_token.rs +++ b/core/simulator/src/workload/ops/delete_personal_access_token.rs @@ -19,7 +19,7 @@ //! (a fabricated name). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -39,7 +39,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::NotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/delete_segments.rs b/core/simulator/src/workload/ops/delete_segments.rs index abbb173ff5..ede2a8f453 100644 --- a/core/simulator/src/workload/ops/delete_segments.rs +++ b/core/simulator/src/workload/ops/delete_segments.rs @@ -27,7 +27,7 @@ //! outcome expansion lands cleanly post-upgrade. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -53,7 +53,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Success]; pub const fn sample( _shadow: &mut Shadow, _outcome: Outcome, - _prng: &mut Xoshiro256Plus, + _prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { // Disabled until shadow grows a topic-to-namespace index. See module docs. diff --git a/core/simulator/src/workload/ops/delete_stream.rs b/core/simulator/src/workload/ops/delete_stream.rs index 97aa7f050c..6e3a78e82d 100644 --- a/core/simulator/src/workload/ops/delete_stream.rs +++ b/core/simulator/src/workload/ops/delete_stream.rs @@ -19,7 +19,7 @@ //! with a fabricated name that was never created. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -39,7 +39,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/delete_topic.rs b/core/simulator/src/workload/ops/delete_topic.rs index 4dfd8241fb..0d2422d2d0 100644 --- a/core/simulator/src/workload/ops/delete_topic.rs +++ b/core/simulator/src/workload/ops/delete_topic.rs @@ -20,7 +20,7 @@ //! fabricated topic). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -41,7 +41,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/delete_user.rs b/core/simulator/src/workload/ops/delete_user.rs index 55c039eb72..718893dc3a 100644 --- a/core/simulator/src/workload/ops/delete_user.rs +++ b/core/simulator/src/workload/ops/delete_user.rs @@ -19,7 +19,7 @@ //! username). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -39,7 +39,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::UserNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/mod.rs b/core/simulator/src/workload/ops/mod.rs index d1aa4b483f..0e77f15e0b 100644 --- a/core/simulator/src/workload/ops/mod.rs +++ b/core/simulator/src/workload/ops/mod.rs @@ -53,8 +53,9 @@ pub mod update_stream; pub mod update_topic; pub mod update_user; -use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use iggy_binary_protocol::{KIND_CONSUMER, KIND_CONSUMER_GROUP, RoutedRequestHeader}; +use rand::RngExt; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -63,6 +64,23 @@ use crate::workload::effect::Effect; use crate::workload::options::WorkloadOptions; use crate::workload::shadow::Shadow; +/// Draw a consumer kind for the four consumer-offset ops, as the WIRE +/// discriminant rather than a bare boolean. +/// +/// `WireConsumer::decode` accepts only [`KIND_CONSUMER`] (1) and +/// [`KIND_CONSUMER_GROUP`] (2). Anything else maps to +/// `IggyError::InvalidCommand`, which the partition plane answers by logging a +/// WARN and dropping the frame with NO reply, so one malformed draw wedges that +/// client's in-flight slot for the rest of the run. One bool draw either way, so +/// the PRNG trace shape is unchanged. +pub(crate) fn sample_consumer_kind(prng: &mut Xoshiro256PlusPlus) -> u8 { + if prng.random::() { + KIND_CONSUMER_GROUP + } else { + KIND_CONSUMER + } +} + /// Generates per-op enums (`InFlightInput`, `InFlightOutcome`) plus four /// dispatch fns over a fixed `(Action, module)` table. Missing variants /// are a compile error via the exhaustive `match` arms. @@ -100,7 +118,7 @@ macro_rules! op_dispatch { pub fn sample( action: Action, shadow: &mut Shadow, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, options: &WorkloadOptions, outcome_id: usize, ) -> Option<(InFlightInput, InFlightOutcome)> { diff --git a/core/simulator/src/workload/ops/purge_stream.rs b/core/simulator/src/workload/ops/purge_stream.rs index a7f89c66d8..19ba334874 100644 --- a/core/simulator/src/workload/ops/purge_stream.rs +++ b/core/simulator/src/workload/ops/purge_stream.rs @@ -19,7 +19,7 @@ //! (fabricated, never-created name). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -39,7 +39,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/purge_topic.rs b/core/simulator/src/workload/ops/purge_topic.rs index 875c625442..969ee1afb8 100644 --- a/core/simulator/src/workload/ops/purge_topic.rs +++ b/core/simulator/src/workload/ops/purge_topic.rs @@ -19,7 +19,7 @@ //! parent stream), or `TopicNotFound` (live stream, fabricated topic). use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -40,7 +40,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/send_messages.rs b/core/simulator/src/workload/ops/send_messages.rs index fda0c06565..79b82c3eba 100644 --- a/core/simulator/src/workload/ops/send_messages.rs +++ b/core/simulator/src/workload/ops/send_messages.rs @@ -24,7 +24,7 @@ use bytes::Bytes; use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use server_common::sharding::IggyNamespace; @@ -51,7 +51,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Success]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/store_consumer_offset.rs b/core/simulator/src/workload/ops/store_consumer_offset.rs index 1f2821bf70..4d021f1834 100644 --- a/core/simulator/src/workload/ops/store_consumer_offset.rs +++ b/core/simulator/src/workload/ops/store_consumer_offset.rs @@ -25,12 +25,13 @@ use iggy_binary_protocol::{AckLevel, RoutedRequestHeader}; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use server_common::sharding::IggyNamespace; use crate::client::SimClient; use crate::workload::effect::Effect; +use crate::workload::ops::sample_consumer_kind; use crate::workload::options::WorkloadOptions; use crate::workload::shadow::Shadow; @@ -53,13 +54,13 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Success]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, options: &WorkloadOptions, ) -> Option { match outcome { Outcome::Success => { let ns = shadow.pick_namespace(prng)?; - let consumer_kind: u8 = u8::from(prng.random::()); + let consumer_kind = sample_consumer_kind(prng); let consumer_id: u32 = prng.random_range(0..options.consumer_pool_size.max(1)); // Draw against the configured ceiling, then clamp to committed // reality so the offset is reachable. Clamping post-draw keeps diff --git a/core/simulator/src/workload/ops/update_permissions.rs b/core/simulator/src/workload/ops/update_permissions.rs index b3aad70e8e..d05a68c20a 100644 --- a/core/simulator/src/workload/ops/update_permissions.rs +++ b/core/simulator/src/workload/ops/update_permissions.rs @@ -19,7 +19,7 @@ //! (fabricated user). No permissions payload, so every outcome is `Effect::None`. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -39,7 +39,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::UserNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/update_stream.rs b/core/simulator/src/workload/ops/update_stream.rs index 0832fa1e9b..c85bccf6fe 100644 --- a/core/simulator/src/workload/ops/update_stream.rs +++ b/core/simulator/src/workload/ops/update_stream.rs @@ -22,7 +22,7 @@ //! targeted, but the server still classifies it on a race. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -43,7 +43,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/update_topic.rs b/core/simulator/src/workload/ops/update_topic.rs index 6df1a03e43..83720e929c 100644 --- a/core/simulator/src/workload/ops/update_topic.rs +++ b/core/simulator/src/workload/ops/update_topic.rs @@ -22,7 +22,7 @@ //! `NameAlreadyExists` not targeted. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -44,7 +44,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/ops/update_user.rs b/core/simulator/src/workload/ops/update_user.rs index 5b13139f3e..b76cbf8d0a 100644 --- a/core/simulator/src/workload/ops/update_user.rs +++ b/core/simulator/src/workload/ops/update_user.rs @@ -19,7 +19,7 @@ //! `UserNotFound` (fabricated user). `UsernameAlreadyExists` not targeted. use iggy_binary_protocol::RoutedRequestHeader; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; use crate::client::SimClient; @@ -44,7 +44,7 @@ pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::UserNotFound]; pub fn sample( shadow: &mut Shadow, outcome: Outcome, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, _options: &WorkloadOptions, ) -> Option { match outcome { diff --git a/core/simulator/src/workload/options.rs b/core/simulator/src/workload/options.rs index 31ab5444df..e8ed70d859 100644 --- a/core/simulator/src/workload/options.rs +++ b/core/simulator/src/workload/options.rs @@ -17,7 +17,25 @@ use crate::workload::actions::Action; use server_common::sharding::IggyNamespace; -use strum::EnumCount; +use strum::{EnumCount, IntoEnumIterator}; + +/// Default [`WorkloadOptions::request_timeout_ticks`]. +/// +/// Four times the primary's commit-broadcast interval +/// (`VsrTimeout::COMMIT_MESSAGE_TICKS`, 50), so a request merely waiting on the next +/// broadcast is never resent, while one lost to a dropped packet or a crashed +/// primary is retried well inside the run budget. +pub const DEFAULT_REQUEST_TIMEOUT_TICKS: u64 = 200; + +/// Default [`WorkloadOptions::crash_stability_ticks`]. Long enough that the +/// surviving primary commits past the crashed replica's log, so its rejoin has +/// something to repair. +pub const DEFAULT_CRASH_STABILITY_TICKS: u64 = 300; + +/// Default [`WorkloadOptions::restart_stability_ticks`]. Long enough for a +/// rejoined replica to catch up before it becomes a crash candidate again, so a +/// run is not entirely half-repaired replicas. +pub const DEFAULT_RESTART_STABILITY_TICKS: u64 = 500; /// Per-action sampling weights as percentages. Unlisted variants default /// to 0 (never picked). Listed weights must sum to 100. @@ -47,6 +65,85 @@ impl ActionWeights { Self { weights } } + /// Partition plane only: writes plus consumer-offset traffic, no metadata + /// mutation. Drains and converges most readily, so it is what a run reaches for + /// when the question is about replication rather than the state machine. + #[must_use] + pub fn partition_only() -> Self { + Self::new(&[ + (Action::SendMessages, 60), + (Action::StoreConsumerOffset, 32), + (Action::DeleteConsumerOffset, 8), + ]) + } + + /// Metadata plane only, weighted so creates outrun deletes and the shadow keeps + /// a live population to sample duplicate- and missing-target outcomes against. + /// `DeleteSegments` is excluded: it resolves against partition state the + /// partition presets build, so it belongs to a mixed run. + #[must_use] + pub fn metadata_only() -> Self { + Self::new(&[ + (Action::CreateStream, 12), + (Action::UpdateStream, 6), + (Action::DeleteStream, 6), + (Action::PurgeStream, 4), + (Action::CreateTopic, 12), + (Action::UpdateTopic, 6), + (Action::DeleteTopic, 6), + (Action::PurgeTopic, 4), + (Action::CreatePartitions, 6), + (Action::DeletePartitions, 4), + (Action::CreateConsumerGroup, 6), + (Action::DeleteConsumerGroup, 4), + (Action::CreateUser, 6), + (Action::UpdateUser, 3), + (Action::DeleteUser, 3), + (Action::ChangePassword, 3), + (Action::UpdatePermissions, 3), + (Action::CreatePersonalAccessToken, 3), + (Action::DeletePersonalAccessToken, 3), + ]) + } + + /// Every action equally likely. Widest op coverage per tick, at the cost of a + /// shallow population per entity kind. + /// + /// `Action::COUNT` does not divide 100, so the first `100 % COUNT` actions carry + /// one extra point. Spread rather than asserting even division, so appending an + /// `Action` never breaks this preset. + /// + /// # Panics + /// If the spread weights do not sum to 100, which means the remainder + /// arithmetic is wrong rather than the caller. + #[must_use] + pub fn uniform() -> Self { + let count = u32::try_from(Action::COUNT).expect("Action::COUNT fits u32"); + // Above 100 actions the base weight floors to 0 and the remainder gives the + // FIRST 100 a weight of 1, leaving the rest at 0: a spread that still sums + // to 100 while contradicting "every action equally likely". The sum check + // below cannot see that, so it is caught here. + assert!( + count <= 100, + "uniform() cannot spread 100 points over {count} actions without \ + silently starving the tail; widen the weight scale first" + ); + let base = 100 / count; + let remainder = 100 % count; + let entries: Vec<(Action, u8)> = Action::iter() + .enumerate() + .map(|(idx, action)| { + let extra = u32::try_from(idx).expect("action index fits u32") < remainder; + let weight = base + u32::from(extra); + ( + action, + u8::try_from(weight).expect("per-action weight is at most 100"), + ) + }) + .collect(); + Self::new(&entries) + } + #[must_use] pub const fn weight(&self, action: Action) -> u8 { self.weights[action as usize] @@ -92,13 +189,38 @@ pub struct WorkloadOptions { pub consumer_pool_size: u32, /// Upper bound on offset carried by `StoreConsumerOffset`. pub max_offset: u64, - /// Probability per tick that the driver crashes one live non-primary - /// replica (crash-only, no restart). `0.0` disables injection: the fault - /// PRNG draws nothing, so traffic stays bit-identical. + /// Probability per tick that the driver crashes one eligible replica. + /// `0.0` disables injection entirely: the fault PRNG draws nothing, so + /// traffic stays bit-identical. pub crash_per_tick_ratio: f32, + /// Probability per tick that the driver restarts one crashed replica. + /// Meaningless without `crash_per_tick_ratio`, since nothing is ever down. + pub restart_per_tick_ratio: f32, + /// Ticks a replica must stay down before it may be restarted. Keeps a crash + /// long enough to actually matter: a replica restarted the tick after it + /// crashed never falls behind, so nothing needs repairing. + pub crash_stability_ticks: u64, + /// Ticks a replica must stay up before it may be crashed again. Stops a + /// single unlucky replica from being crash-looped while its peers never + /// fail. + pub restart_stability_ticks: u64, + /// Leave the primary of every tracked namespace out of the crash pool. + /// + /// Defaults to `true`, which is what the driver did unconditionally before + /// clients could resend: a request lost to a crashed primary was never retried + /// and stranded the client's only in-flight slot. With resending in place, + /// `false` puts a view change under live traffic. + pub spare_primary: bool, /// Floor on live replicas the driver will not crash below, preserving a /// commit quorum. Defaults to `replica_count / 2 + 1`. pub min_survivors: u8, + /// Ticks a request may stay outstanding before the client resends it. + /// + /// Must clear the primary's commit-broadcast interval with room to spare, or the + /// run spends its budget resending work that was about to be answered, and must + /// stay well under the tick budget, or a lost request is never retried and its + /// client's slot strands. `0` disables resending. + pub request_timeout_ticks: u64, } impl WorkloadOptions { @@ -118,7 +240,12 @@ impl WorkloadOptions { consumer_pool_size: 4, max_offset: 1_000_000, crash_per_tick_ratio: 0.0, + restart_per_tick_ratio: 0.0, + crash_stability_ticks: DEFAULT_CRASH_STABILITY_TICKS, + restart_stability_ticks: DEFAULT_RESTART_STABILITY_TICKS, + spare_primary: true, min_survivors: replica_count / 2 + 1, + request_timeout_ticks: DEFAULT_REQUEST_TIMEOUT_TICKS, } } } diff --git a/core/simulator/src/workload/oracle.rs b/core/simulator/src/workload/oracle.rs index b5a2b6ce04..16f6a48ad4 100644 --- a/core/simulator/src/workload/oracle.rs +++ b/core/simulator/src/workload/oracle.rs @@ -23,25 +23,25 @@ //! //! - no live replica is ahead of the leader on any namespace (a backup ahead of //! the leader is a split-brain / divergence bug), +//! - every live replica agrees with every other on each committed metadata op +//! they both hold, the real consensus property (see +//! [`super::state_checker`]), //! - on a serial run, the workload's predicted [`Shadow`] equals the metadata //! committed on the leader, the payoff of the name-keyed shadow. //! -//! Full cross-replica EQUALITY (every live replica holding the same committed -//! log) is the real consensus property, but it is not asserted yet. Backups -//! apply prepares in strict order and drop any gap (`op != current_op + 1` in -//! `metadata::on_replicate` / `iggy_partition`), relying on the primary's -//! retransmit and the repair sessions (`MetadataRepairSession` / partition -//! `RepairSession`) to refill. Message repair has landed on both planes, so -//! the equality assert is unblocked but not yet re-enabled: the sim must -//! first drive quiesce long enough for repair rounds to converge. +//! Equality is asserted over the committed PREFIX, not over equal heads: a replica +//! that missed the last commit broadcast, or rejoined recently, may legitimately +//! trail. What it may not do is hold different history at an op it did commit. +//! Requiring equal heads would fail on ordinary lag and say nothing about safety. use crate::Simulator; use crate::replica::Replica; use crate::workload::shadow::Shadow; -use crate::workload::{Workload, apply_sim_commands}; -use consensus::{MetadataHandle, Status}; +use crate::workload::{Workload, apply_sim_commands, resubmit_due, state_checker}; +use consensus::{Consensus, MetadataHandle, Status}; use metadata::impls::metadata::StreamsFrontend; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; /// Prefix every workload-generated entity name carries (see /// [`Shadow::fresh_name`]). The entity oracle filters committed state to these @@ -72,14 +72,24 @@ struct CommittedMetadata { impl CommittedMetadata { /// Restrict to workload-generated entities (see [`WORKLOAD_PREFIX`]), so the /// entity oracle compares like with like against the shadow. + /// + /// Every level is filtered on its OWN name, not its stream's. The harness seeds + /// filler topics and partitions (`sim-topic-*`, see `Streams::seed_namespace`) + /// to keep slab ids dense, and once the workload has created enough streams + /// those land inside a stream named `wl-...`. Filtering topics by their stream + /// alone then admits harness state and the shadow is blamed for missing it. fn workload_owned(mut self) -> Self { self.streams .retain(|name| name.starts_with(WORKLOAD_PREFIX)); - self.topics - .retain(|(stream, _)| stream.starts_with(WORKLOAD_PREFIX)); + self.topics.retain(|(stream, topic)| { + stream.starts_with(WORKLOAD_PREFIX) && topic.starts_with(WORKLOAD_PREFIX) + }); self.users.retain(|name| name.starts_with(WORKLOAD_PREFIX)); - self.consumer_groups - .retain(|(stream, _, _)| stream.starts_with(WORKLOAD_PREFIX)); + self.consumer_groups.retain(|(stream, topic, group)| { + stream.starts_with(WORKLOAD_PREFIX) + && topic.starts_with(WORKLOAD_PREFIX) + && group.starts_with(WORKLOAD_PREFIX) + }); self } } @@ -96,10 +106,22 @@ impl CommittedMetadata { pub fn drive_to_quiesce(sim: &mut Simulator, workload: &mut Workload, max_ticks: u64) -> bool { let mut drained = false; for _ in 0..max_ticks { + // The drain keeps resending: a request lost on the way out is never + // answered, so without retries the drain would spend its whole budget + // waiting on a reply that cannot arrive. + workload.tick(); + resubmit_due(sim, workload); for reply in sim.step() { let cmds = workload.on_reply(&reply); apply_sim_commands(sim, &cmds); } + // A resend can land on a transport a restart left unbound, which the server + // refuses with an eviction. No re-login here, unlike the active driver: the + // drain submits nothing new, and the refused request was rejected before + // commit, so forgetting it is what "drained" means. + for client_id in sim.take_evictions() { + workload.forget_evicted_client(client_id); + } if workload.total_in_flight() == 0 { drained = true; break; @@ -117,29 +139,227 @@ pub fn drive_to_quiesce(sim: &mut Simulator, workload: &mut Workload, max_ticks: true } -/// Post-drain consensus checks that hold today. +/// Why the run did not drain, as a multi-line report. /// -/// Asserts no live replica is ahead of the leader, and (on a serial run) that -/// the shadow equals the metadata committed on the leader. See the module docs -/// for why full cross-replica equality is deferred. +/// A failed drain is either a wedge or a merely slow cluster, and the bare boolean +/// [`drive_to_quiesce`] returns cannot tell them apart. With crashes, restarts and +/// packet loss in play that distinction is the whole diagnosis, so name what is +/// outstanding and what every live replica believes. +#[must_use] +pub fn quiesce_failure_report(sim: &Simulator, workload: &Workload) -> String { + let mut report = format!( + "did not drain: {} request(s) still outstanding (seed={:#x})\n", + workload.total_in_flight(), + workload.options.seed, + ); + for row in workload.outstanding_summary() { + let _ = writeln!( + report, + " outstanding client={} request={} action={:?} last_target=replica {} \ + attempts={}", + row.client, row.request, row.action, row.target, row.attempts, + ); + } + let _ = writeln!(report, " resends issued: {}", workload.resends()); + for replica_idx in 0..sim.replica_count { + if sim.is_crashed(replica_idx) { + let _ = writeln!(report, " replica {replica_idx}: CRASHED"); + continue; + } + let _ = write!(report, " replica {replica_idx}: live"); + // Metadata plane first: a rejoining replica is quorum-invisible until its + // view probe completes, so its status separates "the cluster is slow" from + // "the cluster has no quorum despite enough live replicas". + if let Some(consensus) = sim.replicas[usize::from(replica_idx)].shards[0] + .plane + .metadata() + .consensus + .as_ref() + { + // The three ways `is_caught_up_primary` stays shut on what otherwise + // reads as a healthy Normal primary, silently dropping every request. + let _ = write!( + report, + " | metadata status={:?} view={} log_view={} commit={}..{} barrier={} \ + transferring={} primary={}", + consensus.status(), + consensus.view(), + consensus.log_view(), + consensus.commit_min(), + consensus.commit_max(), + consensus.recovery_barrier(), + consensus.is_transferring(), + consensus.is_primary(), + ); + } + for &ns in &workload.options.namespaces { + let view = sim.consensus_view(usize::from(replica_idx), ns); + let commit = sim + .offsets(usize::from(replica_idx), ns) + .map(|offsets| offsets.commit_offset); + let primary = sim.primary_index(ns); + let _ = write!( + report, + " | ns {ns:?} view={view:?} commit_offset={commit:?} primary={primary:?}", + ); + } + report.push('\n'); + // Per-client table state. `check_request` admits anything above the + // watermark, so the watermark says whether an outstanding request is still + // expected (above it) or already answered and owed a cached-reply replay (at + // or below it). + let table = sim.replicas[usize::from(replica_idx)].shards[0] + .plane + .metadata() + .client_table + .borrow(); + for client_id in table.client_ids() { + let _ = writeln!( + report, + " client {client_id}: watermark={:?} epoch={:?} cached_reply_request={:?}", + table.get_watermark(client_id), + table.get_epoch(client_id), + table + .get_reply(client_id) + .map(|reply| reply.header().request), + ); + } + } + report +} + +/// Step until every live replica's metadata plane is `Normal` in one shared +/// view, or `max_ticks` elapses. +/// +/// Needed before [`assert_converged`], which resolves the leader as "the live +/// replica whose metadata consensus says it is primary". With primaries spared +/// that was always the same replica in view 0. Once a primary can be crashed, live +/// replicas transiently hold different views and there may be no `Normal` primary +/// at all, so the leader lookup fails or names a deposed one: a false failure, not +/// a divergence. +/// +/// Returns `false` if the views never converge, which is a real liveness failure +/// the caller should report rather than assert against an unsettled cluster. +#[must_use] +pub fn settle_to_stable_view(sim: &mut Simulator, workload: &mut Workload, max_ticks: u64) -> bool { + for _ in 0..max_ticks { + if views_are_settled(sim, workload) { + return true; + } + workload.tick(); + resubmit_due(sim, workload); + for reply in sim.step() { + let cmds = workload.on_reply(&reply); + apply_sim_commands(sim, &cmds); + } + } + views_are_settled(sim, workload) +} + +/// Both planes settled: the metadata group and every tracked partition group. /// -/// Assumes one stable primary that every live replica agrees on: the leader is -/// `Simulator::primary_index` (a single replica's view), and both checks treat -/// it as the authoritative, most-advanced log. Sound today because the driver -/// spares primaries from crashes, so no view change runs mid-test. Once -/// primary-crash injection lands, live replicas can hold different views and -/// this breaks: it may pick a stale or crashed leader (a correctly-ahead new -/// primary then trips "exceeds leader"), or find no `Normal` primary mid-view -/// change (`metadata_leader` returns `None`). Both are false failures. Fix -/// then: resolve the leader by highest `(view, commit_offset)`, or quiesce -/// until live replicas reconverge to one view before asserting. Crash injection -/// already runs but spares primaries (`maybe_inject_crash`); this is deferred -/// until primary-crash injection lands, itself gated on a request-resend path. +/// The partition half matters for the leader-relative check in +/// [`assert_converged`], whose leader comes from `Simulator::primary_index`, a +/// single replica's view of that group's primary. Each partition group runs its own +/// view change, so settling only the metadata plane leaves that check asserting +/// against a deposed leader, and a correctly-ahead new one trips "exceeds leader". +fn views_are_settled(sim: &Simulator, workload: &Workload) -> bool { + if !metadata_view_is_settled(sim) { + return false; + } + workload + .options + .namespaces + .iter() + .all(|&ns| partition_view_is_settled(sim, ns)) +} + +/// True when every live replica's metadata consensus is `Normal` in the same +/// view and exactly one of them claims to be primary. +fn metadata_view_is_settled(sim: &Simulator) -> bool { + let mut view = None; + let mut primaries = 0usize; + let mut live = 0usize; + for replica_idx in 0..sim.replica_count { + if sim.is_crashed(replica_idx) { + continue; + } + let Some(consensus) = sim.replicas[usize::from(replica_idx)].shards[0] + .plane + .metadata() + .consensus + .as_ref() + else { + continue; + }; + live += 1; + if consensus.status() != Status::Normal { + return false; + } + match view { + Some(agreed) if agreed != consensus.view() => return false, + Some(_) => {} + None => view = Some(consensus.view()), + } + if consensus.is_primary() { + primaries += 1; + } + } + live > 0 && primaries == 1 +} + +/// True when every live replica hosting `ns` has that group `Normal` in one +/// shared view with exactly one primary. +/// +/// A replica not hosting the namespace is skipped rather than counted as +/// disagreement: a group materialises only on its hash-owning shard, and a group +/// whose stream the workload deleted is not re-materialised after a restart. +fn partition_view_is_settled(sim: &Simulator, ns: server_common::sharding::IggyNamespace) -> bool { + let mut view = None; + let mut primaries = 0usize; + let mut hosts = 0usize; + for replica_idx in 0..sim.replica_count { + if sim.is_crashed(replica_idx) { + continue; + } + let Some(state) = sim.partition_consensus_state(usize::from(replica_idx), ns) else { + continue; + }; + hosts += 1; + if state.status != Status::Normal { + return false; + } + match view { + Some(agreed) if agreed != state.view => return false, + Some(_) => {} + None => view = Some(state.view), + } + if state.is_primary { + primaries += 1; + } + } + // No live host counts as settled: the offset check has no leader to resolve + // either, and skips the namespace for the same reason. + hosts == 0 || primaries == 1 +} + +/// Post-drain consensus checks. +/// +/// Asserts no live replica is ahead of the leader, that every live replica agrees +/// with every other on each committed metadata op they share, and (on a serial +/// run) that the shadow equals the metadata committed on the leader. +/// +/// Assumes one stable primary every live replica agrees on, which +/// [`settle_to_stable_view`] establishes and callers should run first once +/// primaries can be crashed. Without it this may pick a deposed leader (a +/// correctly-ahead new primary then trips "exceeds leader") or find no `Normal` +/// primary mid-view-change. Both are false failures. /// /// # Panics -/// If a replica is ahead of the leader or the shadow mismatches the leader. The -/// workload seed is in the message so the failing run replays deterministically. -pub fn assert_converged(sim: &Simulator, workload: &Workload) { +/// If a replica is ahead of the leader, two replicas disagree on a committed op, +/// or the shadow mismatches the leader. The workload seed is in the message so the +/// failing run replays deterministically. +pub fn assert_converged(sim: &Simulator, workload: &mut Workload) -> ConvergenceReport { let seed = workload.options.seed; let live: Vec = (0..sim.replica_count) .filter(|replica_idx| !sim.is_crashed(*replica_idx)) @@ -150,44 +370,216 @@ pub fn assert_converged(sim: &Simulator, workload: &Workload) { "no live replicas at quiesce (seed={seed:#x})" ); - // Safety direction: no live replica may be ahead of the leader on any - // namespace. A backup may trail (no idle catch-up yet, see module docs), - // but a backup whose commit_offset exceeds the leader's is a divergence. + // The consensus property proper: replicas compared against each other rather + // than against the workload's expectations. First, because a genuine divergence + // explains any leader confusion below it. + let ops_compared = state_checker::assert_committed_prefixes_agree(sim, seed); + tracing::info!( + ops_compared, + "committed metadata prefixes agree across every live replica" + ); + + let Some(leader) = metadata_leader(sim, &live) else { + panic!("no metadata leader live at quiesce (seed={seed:#x})"); + }; + + // Settlement treats "no live replica hosts this group" as settled, so losing + // every instance of a LIVE partition reads as converged. The metadata read + // separates that from a group whose stream the workload deleted. + let namespaces_checked = assert_live_namespaces_have_a_primary(sim, workload, leader, seed); + + // Safety direction: no live replica may have COMMITTED more of a group than + // its leader has. A backup may trail (no idle catch-up yet, see module docs), + // but a backup committed past the leader is a divergence. + // + // Measured on the group's consensus `commit_min`, not + // `PartitionOffsets::commit_offset`. The latter is the highest durably persisted + // message offset, which counts an uncommitted suffix: a backup that persisted op + // N while the electing view settled on N-1 is ordinary VSR, not divergence. That + // stayed invisible only while primaries were spared, a never-crashed primary + // being always furthest ahead; with primary crashes it fires on a correct + // cluster. for &ns in &workload.options.namespaces { - let Some(leader) = sim.primary_index(ns) else { + let Some(partition_leader) = sim.primary_index(ns) else { continue; }; - let Some(leader_offset) = sim - .offsets(usize::from(leader), ns) - .map(|o| o.commit_offset) + let Some(leader_committed) = sim + .partition_consensus_state(usize::from(partition_leader), ns) + .map(|state| state.commit_min) else { continue; }; for &replica_idx in &live { - if let Some(offset) = sim.offsets(replica_idx, ns).map(|o| o.commit_offset) { + if let Some(committed) = sim + .partition_consensus_state(replica_idx, ns) + .map(|state| state.commit_min) + { assert!( - offset <= leader_offset, - "replica {replica_idx} commit_offset {offset} exceeds leader {leader} \ - ({leader_offset}) on ns {ns:?} at quiesce (seed={seed:#x})", + committed <= leader_committed, + "replica {replica_idx} committed {committed} ops exceeds leader \ + {partition_leader} ({leader_committed}) on ns {ns:?} at quiesce \ + (seed={seed:#x})", ); } } } + let replicas_compared = assert_committed_metadata_agrees(sim, &live, seed); + + let report = ConvergenceReport { + ops_compared, + replicas_compared, + namespaces_checked, + }; + // Entity oracle: on a serial run the shadow must equal the committed // metadata on the leader, the authoritative holder of the metadata log. + // + // Runs even when the oracle is disarmed, because a disarmed oracle is an + // UNKNOWN and this is the measurement that resolves it. Equal means the + // forgotten request's effect is accounted for and the oracle re-arms; unequal + // means the unknown did cost the shadow an effect, which is reported rather than + // asserted, since a failure there would blame the harness's gap on the cluster. + if !workload.serial_run() { + return report; + } + let committed = read_committed_metadata(&sim.replicas[leader].shards[0]).workload_owned(); + let shadow = shadow_metadata(&workload.shadow); if workload.strict_outcome_oracle() { - let Some(leader) = metadata_leader(sim, &live) else { - panic!("no metadata leader live at quiesce (seed={seed:#x})"); - }; - let committed = read_committed_metadata(&sim.replicas[leader].shards[0]).workload_owned(); assert_eq!( - shadow_metadata(&workload.shadow), - committed, + shadow, committed, "shadow diverged from leader-committed metadata at quiesce \ (leader={leader}, seed={seed:#x})", ); + return report; + } + if shadow == committed { + workload.rearm_outcome_oracle(); + tracing::info!( + leader, + "entity oracle re-armed: the shadow matches leader-committed metadata \ + again despite an earlier eviction" + ); + } else { + tracing::warn!( + leader, + "entity oracle stayed disarmed: an evicted client's forgotten request \ + left the shadow and leader-committed metadata unequal, so this run \ + proved nothing about entity state" + ); + } + report +} + +/// What [`assert_converged`] actually managed to compare. +/// +/// Counts of exercised comparisons, not of checks attempted: zero exactly when the +/// property was never tested. An oracle that compared nothing passes like one that +/// compared everything, so callers assert on these. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ConvergenceReport { + /// Committed metadata ops witnessed by more than one live replica. + pub ops_compared: usize, + /// Live replicas whose committed metadata CONTENT was compared against a peer + /// sharing its commit point. Zero on a solo cluster. + pub replicas_compared: usize, + /// Namespaces still present in committed metadata that were required to have a + /// host and exactly one primary. + pub namespaces_checked: usize, +} + +/// Require a host and exactly one primary for every workload namespace committed +/// metadata still knows about. Returns how many namespaces that covered. +/// +/// A namespace the leader no longer holds is skipped: a deleted stream is not +/// re-materialised, and that is the only legitimate reason to have no host. +fn assert_live_namespaces_have_a_primary( + sim: &Simulator, + workload: &Workload, + leader: usize, + seed: u64, +) -> usize { + let streams = sim.replicas[leader].shards[0] + .plane + .metadata() + .mux_stm + .streams(); + let mut checked = 0; + for &ns in &workload.options.namespaces { + if streams.created_revision_for_namespace(ns).is_none() { + continue; + } + checked += 1; + let mut hosts = 0usize; + let mut primaries = 0usize; + for replica_idx in 0..sim.replica_count { + if sim.is_crashed(replica_idx) { + continue; + } + if let Some(state) = sim.partition_consensus_state(usize::from(replica_idx), ns) { + hosts += 1; + if state.is_primary { + primaries += 1; + } + } + } + assert!( + hosts > 0, + "ns {ns:?} is live on leader {leader} but no live replica hosts it at \ + quiesce: every instance was lost (seed={seed:#x})" + ); + assert_eq!( + primaries, 1, + "ns {ns:?} is live on leader {leader} but {hosts} host(s) report \ + {primaries} primaries at quiesce (seed={seed:#x})" + ); + } + checked +} + +/// Assert live replicas standing at the same committed op hold the same committed +/// metadata. Returns how many replicas were compared against a peer. +/// +/// The gap [`state_checker`] leaves by construction: it compares prepare HEADERS, so +/// two replicas whose logs agree op for op pass even when one applied that log onto a +/// different baseline, which is what a botched restart leaves. +/// +/// Grouped by commit point because a replica that trails legitimately holds less. +/// Workload-owned only because `seed_stream_topic_partition` seeds the `sim-*` filler +/// straight into each LIVE replica's STM, bypassing consensus, so those differ across +/// replicas on a healthy cluster. +fn assert_committed_metadata_agrees(sim: &Simulator, live: &[usize], seed: u64) -> usize { + let mut by_commit: BTreeMap = BTreeMap::new(); + let mut compared = 0; + for &replica_idx in live { + let Some(consensus) = sim.replicas[replica_idx].shards[0] + .plane + .metadata() + .consensus + .as_ref() + else { + continue; + }; + let committed = + read_committed_metadata(&sim.replicas[replica_idx].shards[0]).workload_owned(); + match by_commit.get(&consensus.commit_min()) { + Some((owner, canonical)) => { + assert_eq!( + &committed, + canonical, + "at quiesce replicas {owner} and {replica_idx} both committed \ + through metadata op {} but hold different metadata: one applied \ + that log onto a different baseline (seed={seed:#x})", + consensus.commit_min(), + ); + compared += 1; + } + None => { + by_commit.insert(consensus.commit_min(), (replica_idx, committed)); + } + } } + compared } /// The live replica whose metadata consensus is the current primary, i.e. the diff --git a/core/simulator/src/workload/shadow.rs b/core/simulator/src/workload/shadow.rs index 6a34207fe6..2692ff1684 100644 --- a/core/simulator/src/workload/shadow.rs +++ b/core/simulator/src/workload/shadow.rs @@ -28,7 +28,7 @@ use indexmap::IndexSet; use rand::RngExt; -use rand_xoshiro::Xoshiro256Plus; +use rand_xoshiro::Xoshiro256PlusPlus; use std::collections::HashMap; use server_common::sharding::IggyNamespace; @@ -99,7 +99,7 @@ impl Shadow { /// Pick a live namespace uniformly. `IndexSet` preserves insertion /// order, so for deduplicated input this matches `Vec::get(i)`. - pub fn pick_namespace(&self, prng: &mut Xoshiro256Plus) -> Option { + pub fn pick_namespace(&self, prng: &mut Xoshiro256PlusPlus) -> Option { let n = self.namespaces_live.len(); if n == 0 { return None; @@ -108,7 +108,7 @@ impl Shadow { self.namespaces_live.get_index(i).copied() } - pub fn pick_stream_name(&self, prng: &mut Xoshiro256Plus) -> Option { + pub fn pick_stream_name(&self, prng: &mut Xoshiro256PlusPlus) -> Option { let n = self.stream_names.len(); if n == 0 { return None; @@ -117,7 +117,7 @@ impl Shadow { self.stream_names.get_index(i).cloned() } - pub fn pick_topic_pair(&self, prng: &mut Xoshiro256Plus) -> Option<(String, String)> { + pub fn pick_topic_pair(&self, prng: &mut Xoshiro256PlusPlus) -> Option<(String, String)> { let n = self.topic_names.len(); if n == 0 { return None; @@ -126,7 +126,7 @@ impl Shadow { self.topic_names.get_index(i).cloned() } - pub fn pick_user_name(&self, prng: &mut Xoshiro256Plus) -> Option { + pub fn pick_user_name(&self, prng: &mut Xoshiro256PlusPlus) -> Option { let n = self.user_names.len(); if n == 0 { return None; @@ -135,7 +135,7 @@ impl Shadow { self.user_names.get_index(i).cloned() } - pub fn pick_pat_name(&self, prng: &mut Xoshiro256Plus) -> Option { + pub fn pick_pat_name(&self, prng: &mut Xoshiro256PlusPlus) -> Option { let n = self.pat_names.len(); if n == 0 { return None; @@ -146,7 +146,7 @@ impl Shadow { pub fn pick_consumer_group_triple( &self, - prng: &mut Xoshiro256Plus, + prng: &mut Xoshiro256PlusPlus, ) -> Option<(String, String, String)> { let n = self.consumer_group_names.len(); if n == 0 { diff --git a/core/simulator/src/workload/state_checker.rs b/core/simulator/src/workload/state_checker.rs new file mode 100644 index 0000000000..ded5086516 --- /dev/null +++ b/core/simulator/src/workload/state_checker.rs @@ -0,0 +1,435 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Cross-replica committed-log equality. +//! +//! The per-tick checks in [`super::invariants`] catch a single replica +//! contradicting itself, and [`super::oracle`] compares committed metadata against +//! the workload's shadow. Neither compares replicas to EACH OTHER, which is the +//! actual consensus property: two replicas that both committed op N must have +//! committed the same op N. +//! +//! Keeps one canonical commit chain and asserts every replica agrees with it +//! wherever they overlap, in BOTH directions of `(commit_a == commit_b) == +//! (checksum_a == checksum_b)`: same op implies same prepare, and same prepare +//! implies same op, the second of which catches one prepare committing at two log +//! positions. Also asserts the chain is hash-linked (`header_b.parent == +//! checksum_a`). Recording which replicas reached each op keeps the check provably +//! non-vacuous: a chain nothing was compared against passes silently. + +use crate::Simulator; +use consensus::MetadataHandle; +use iggy_binary_protocol::PrepareHeader; +use journal::Journal; +use std::collections::{BTreeMap, BTreeSet}; + +/// One op of the canonical committed chain. +#[derive(Debug)] +struct CanonicalCommit { + /// Identity of the prepare committed at this op. Two replicas disagreeing here + /// is a divergence: the same log position holds different history. + /// + /// The hash link is checked against this rather than a stored `parent`: an + /// arriving header's `parent` must equal the canonical previous op's `checksum`, + /// so keeping each entry's own parent would record a value nothing reads. + checksum: u128, + /// Replicas observed committing this op, so the check can prove it compared + /// something rather than passing over an empty chain. + replicas: BTreeSet, +} + +/// Canonical committed metadata chain, accumulated across ticks. +#[derive(Debug, Default)] +pub struct StateChecker { + commits: BTreeMap, + /// Op each prepare checksum was committed at, the reverse half of `(commit_a == + /// commit_b) == (checksum_a == checksum_b)`. `commits` alone gives only the + /// forward half (same op implies same checksum); without this, the same prepare + /// appearing at two log positions (a duplicate apply, a misnumbered replay) is + /// invisible. + ops_by_checksum: BTreeMap, + /// Each replica's commit point as of the last check, so a tick only walks what + /// is new. + /// + /// LOWERED when a replica's commit point drops, which a restart does: the point + /// is recovered from a lower bound (`SimJournal::recovery_commit_watermark`), so + /// it re-commits a range it already reported. A high-water mark here would + /// compare those re-commits against nothing, exactly the syncing case this check + /// exists for. The cost is re-walking a recovering replica's prefix, bounded by + /// its own commit point. + verified_upto: BTreeMap, + /// Each replica's metadata incarnation as of the last check. + /// + /// A change means everything below the commit point was rebuilt from disk, so it + /// is not the bytes already compared. Lowering the mark alone misses this: a + /// replica recovering the SAME point leaves an empty range to walk. + incarnations: BTreeMap, +} + +impl StateChecker { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Fold every live replica's newly committed metadata ops into the canonical + /// chain, asserting agreement. + /// + /// Committed state only (ops at or below `commit_min`), so a prepare still in + /// flight is never compared: replicas may disagree about uncommitted tails, and + /// that is what a view change resolves. Crashed replicas are skipped rather than + /// dropped, keeping their mark, so a restart re-verifies only what it commits + /// anew. + /// + /// # Panics + /// On any disagreement about a committed op, or a broken hash chain. The + /// message names both replicas and the op, and the seed replays the run. + pub fn check(&mut self, sim: &Simulator, seed: u64) { + for replica_idx in 0..sim.replica_count { + if sim.is_crashed(replica_idx) { + continue; + } + let replica = &sim.replicas[usize::from(replica_idx)]; + let Some(consensus) = replica.shards[0].plane.metadata().consensus.as_ref() else { + continue; + }; + let committed = consensus.commit_min(); + // Re-walk from the snapshot floor, which bounds the cost: below it the + // state came from the snapshot and has no header to compare. + let restarted = self + .incarnations + .insert(replica_idx, replica.metadata_incarnation) + .is_some_and(|previous| previous != replica.metadata_incarnation); + if restarted { + self.verified_upto + .insert(replica_idx, replica.metadata_journal.snapshot_op()); + } + let verified = self.verified_upto.get(&replica_idx).copied().unwrap_or(0); + for op in (verified + 1)..=committed { + self.verify(replica_idx, replica, op, committed, seed); + } + // Again, even when the loop already walked it. A replica whose point does + // not move is otherwise never compared, while what it holds there can + // change under it (recovery, repair, transfer install). TigerBeetle + // re-derives the checksum at `commit_min` every tick for this reason. + if committed > 0 { + self.verify(replica_idx, replica, committed, committed, seed); + } + // Recorded verbatim, NOT maxed: see the field doc. + self.verified_upto.insert(replica_idx, committed); + } + } + + /// Fold one replica's op into the canonical chain, or prove its absence + /// legitimate. + fn verify( + &mut self, + replica_idx: u8, + replica: &crate::SimReplica, + op: u64, + committed: u64, + seed: u64, + ) { + let Some(header) = journaled_header(replica, op) else { + assert!( + absent_header_is_legitimate(replica, op), + "replica {replica_idx} reports op {op} committed (point {committed}) \ + but holds no header for it above its snapshot floor {}: a hole in the \ + committed log (seed={seed:#x})", + replica.metadata_journal.snapshot_op(), + ); + return; + }; + self.record(replica_idx, op, &header, seed); + } + + /// Number of ops in the canonical chain. Tests assert this is non-zero, so a + /// green run cannot mean "never compared anything". + #[must_use] + pub fn chain_len(&self) -> usize { + self.commits.len() + } + + /// Ops witnessed by more than one replica, the only ones that exercised the + /// equality property: an op seen on a single replica was recorded, not compared. + #[must_use] + pub fn ops_compared(&self) -> usize { + self.commits + .values() + .filter(|commit| commit.replicas.len() > 1) + .count() + } + + fn record(&mut self, replica_idx: u8, op: u64, header: &PrepareHeader, seed: u64) { + // Hash-chain link, checked before the identity comparison so a diverged + // prefix is reported at the op where the chains part rather than at the + // first op whose contents happen to differ. + if let Some(previous) = self.commits.get(&(op - 1)) + && header.parent != previous.checksum + { + panic!( + "replica {replica_idx} committed op {op} whose parent {:#x} is not the \ + canonical op {} checksum {:#x}: its committed history forked below this \ + op (seed={seed:#x})", + header.parent, + op - 1, + previous.checksum, + ); + } + // The reverse half: one prepare, one log position. A checksum turning up at + // a second op means the same prepare committed twice, which per-op agreement + // alone would never show. + match self.ops_by_checksum.get(&header.checksum) { + Some(&previous_op) => assert_eq!( + previous_op, op, + "replica {replica_idx} committed the prepare with checksum {:#x} at op \ + {op}, but it is already the canonical commit at op {previous_op}: one \ + prepare committed at two log positions (seed={seed:#x})", + header.checksum, + ), + None => { + self.ops_by_checksum.insert(header.checksum, op); + } + } + match self.commits.get_mut(&op) { + Some(canonical) => { + assert_eq!( + canonical.checksum, header.checksum, + "replicas disagree on committed op {op}: canonical checksum {:#x} \ + (committed by {:?}) vs replica {replica_idx}'s {:#x}. Two replicas \ + committed different history at the same log position (seed={seed:#x})", + canonical.checksum, canonical.replicas, header.checksum, + ); + canonical.replicas.insert(replica_idx); + } + None => { + self.commits.insert( + op, + CanonicalCommit { + checksum: header.checksum, + replicas: BTreeSet::from([replica_idx]), + }, + ); + } + } + } +} + +/// Assert every live replica's committed metadata prefix agrees, op for op. +/// +/// The quiesce-time counterpart to [`StateChecker::check`]: that one folds ops in +/// as they commit and compares whatever overlaps, while this walks the full +/// committed prefix of every live replica at rest and requires the shorter to be a +/// genuine PREFIX of the longer. A replica may trail, having missed the last commit +/// broadcast, but where it committed anything it must match contiguously. +/// +/// Returns how many ops were witnessed by MORE THAN ONE replica, i.e. how many +/// exercised the property. Zero means this proved nothing, so a caller that wants +/// the check to mean something asserts on it: a walk that compared nothing passes +/// exactly like one that compared everything. +/// +/// # Panics +/// If two live replicas disagree on any committed op, or a replica's committed +/// prefix has a hole in it. +#[must_use] +pub fn assert_committed_prefixes_agree(sim: &Simulator, seed: u64) -> usize { + let mut canonical: BTreeMap = BTreeMap::new(); + let mut witnesses: BTreeMap = BTreeMap::new(); + for replica_idx in 0..sim.replica_count { + if sim.is_crashed(replica_idx) { + continue; + } + let replica = &sim.replicas[usize::from(replica_idx)]; + let Some(consensus) = replica.shards[0].plane.metadata().consensus.as_ref() else { + continue; + }; + let committed = consensus.commit_min(); + for op in 1..=committed { + let Some(header) = journaled_header(replica, op) else { + // A PREFIX is what this claims to compare, so a hole cannot be + // stepped over: skipping it would let a replica missing ops 5..9 + // pass by agreeing on 1..4 and 10.., the shape a bad repair leaves. + // The one legitimate absence is an op dropped under the snapshot + // floor. + assert!( + absent_header_is_legitimate(replica, op), + "at quiesce replica {replica_idx} reports op {op} committed but holds \ + no header for it (snapshot floor {}, commit point {committed}): its \ + committed prefix has a hole (seed={seed:#x})", + replica.metadata_journal.snapshot_op(), + ); + continue; + }; + if let Some(&(checksum, owner)) = canonical.get(&op) { + assert_eq!( + checksum, header.checksum, + "at quiesce replica {replica_idx} and replica {owner} disagree on \ + committed metadata op {op}: {:#x} vs {checksum:#x} (seed={seed:#x})", + header.checksum, + ); + *witnesses.entry(op).or_insert(1) += 1; + } else { + canonical.insert(op, (header.checksum, replica_idx)); + witnesses.insert(op, 1); + } + } + } + witnesses.values().filter(|&&count| count > 1).count() +} + +/// Whether a committed op having no journal header is legitimate rather than a +/// hole. +/// +/// One case only: the journal dropped it under its own snapshot floor, which a +/// checkpoint does. +/// +/// The commit point itself used to be exempt too, on the grounds that +/// `recovery_commit_watermark` is a lower bound. Unnecessary and unsafe: a checkpoint +/// drains `0..=snapshot_op - 1`, retaining the commit-point header for view-change +/// merging (`checkpoint_drain_retains_the_commit_point_header`), so on a healthy +/// replica it is always there and the exemption only hid a missing committed head. +/// TigerBeetle draws the same line at `commit_min == op_checkpoint()`. +fn absent_header_is_legitimate(replica: &crate::SimReplica, op: u64) -> bool { + op <= replica.metadata_journal.snapshot_op() +} + +/// The header a replica has journaled at `op`, if any. +/// +/// Reads shard 0's retained metadata WAL, which is where the committed metadata +/// log lives; the journal is harness-owned so this also works across a restart. +fn journaled_header(replica: &crate::SimReplica, op: u64) -> Option { + let slot = usize::try_from(op).ok()?; + replica.metadata_journal.header(slot).copied() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client::SimClient; + use crate::packet::PacketSimulatorOptions; + + const SEED: u64 = 0x5C11; + + /// A three-replica cluster with six committed metadata ops behind it. + fn cluster_with_committed_ops() -> Simulator { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + let client_id: u128 = 1; + let mut sim = Simulator::new( + 3, + std::iter::once(client_id), + PacketSimulatorOptions { + node_count: 3, + client_count: 1, + seed: SEED, + ..PacketSimulatorOptions::default() + }, + ); + let client = SimClient::new(client_id); + sim.register_client_with_primary(&client); + for sequence in 0..6u32 { + let msg = client.create_stream(&format!("wl-chk-{sequence}")); + sim.submit_request(client_id, 0, msg.into_generic()); + for _ in 0..40 { + sim.step(); + } + } + sim + } + + /// A committed op with no header is a hole, including at the commit point, which + /// is the one op a checkpoint deliberately retains. + #[test] + #[should_panic(expected = "a hole in the committed log")] + fn a_missing_committed_head_above_the_snapshot_floor_is_a_hole() { + let sim = cluster_with_committed_ops(); + let committed = sim.replicas[1].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .expect("shard 0 owns metadata consensus") + .commit_min(); + assert!( + committed > sim.replicas[1].metadata_journal.snapshot_op(), + "the commit point must sit above the snapshot floor or this test is vacuous" + ); + assert!( + sim.replicas[1].metadata_journal.forget_op(committed), + "the head this test removes must have been there" + ); + StateChecker::new().check(&sim, SEED); + } + + /// A hole punched below a replica's verified mark is caught after it restarts. + /// + /// The mark survives a crash, so without the incarnation reset the walk begins + /// past the damage. The hole stands in for any way a rebuilt prefix can differ + /// from the one already compared. + #[test] + #[should_panic(expected = "a hole in the committed log")] + fn a_damaged_prefix_is_rechecked_after_a_restart() { + let mut sim = cluster_with_committed_ops(); + let mut checker = StateChecker::new(); + checker.check(&sim, SEED); + assert!( + checker.chain_len() > 2, + "the checker verified nothing, so the mark it carries into the restart \ + proves nothing" + ); + + sim.replica_crash(1); + sim.replica_restart(1); + for _ in 0..200 { + sim.step(); + } + assert!( + sim.replicas[1].metadata_journal.forget_op(2), + "op 2 must be journaled for this test to damage anything" + ); + checker.check(&sim, SEED); + } + + /// An undamaged prefix passes the recheck: the reset must turn a restart into a + /// comparison, not into a failure. + #[test] + fn an_undamaged_prefix_passes_the_recheck_after_a_restart() { + let mut sim = cluster_with_committed_ops(); + let mut checker = StateChecker::new(); + checker.check(&sim, SEED); + let before = checker.chain_len(); + + sim.replica_crash(1); + sim.replica_restart(1); + for _ in 0..400 { + sim.step(); + } + checker.check(&sim, SEED); + assert!( + checker.chain_len() >= before, + "the canonical chain shrank across a restart" + ); + assert!( + checker.ops_compared() > 0, + "no op was witnessed by more than one replica, so the recheck compared \ + nothing" + ); + } +}