From 167b443d7036ac5aa5bbe19702da045742bd7f1a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 19:29:25 +0000 Subject: [PATCH 1/4] feat(testing): add Any factory for arbitrary seedable test values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce `Any`, a first-party, dependency-free source of valid-but-arbitrary values for the inputs a test needs yet never asserts on (error codes, messages, instants, ids, enums, primitives). An explicit `Any` call reads as "this value is incidental" where a hand-picked literal reads as "this matters". The source is stored in an AsyncLocal, so it flows with the execution context and is safe under parallel test runners — the same contract the clock and instance-id seams already keep. Values are arbitrary by default (which exposes tests that secretly depend on one) and become deterministic inside an `Any.UseSeed(...)` scope for reproducibility. The clock and instance-id seams gain matching `UseAny()` variants layered on the same source: `Clock.UseAny()` freezes OccurredAt to one arbitrary instant, and `InstanceIds.UseAny()` assigns a distinct arbitrary id per error; both take an optional seed. ErrorContextKey is deliberately left out of this first surface: its process-wide registry has no public reset, so minting arbitrary keys would accumulate global state. The decision and its reasoning are recorded in ADR-0006 (Proposed). - FirstClassErrors.Testing: Any, Clock.UseAny, InstanceIds.UseAny - Tests covering seeding determinism, scope isolation, sentinel exclusion - Testing guide updated in English and French, plus the NuGet readme - ADR-0006 proposed and indexed Refs: #135 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012ULkMmGS2EgnALAJtwpxtP --- FirstClassErrors.Testing/Any.cs | 259 ++++++++++++++++++ FirstClassErrors.Testing/Clock.cs | 26 ++ FirstClassErrors.Testing/InstanceIds.cs | 27 ++ FirstClassErrors.Testing/README.nuget.md | 3 + FirstClassErrors.UnitTests/AnyTests.cs | 127 +++++++++ .../ClockUseAnyTests.cs | 56 ++++ .../InstanceIdsUseAnyTests.cs | 69 +++++ doc/DeterministicTesting.en.md | 55 ++++ doc/DeterministicTesting.fr.md | 55 ++++ ...rary-test-values-from-a-seedable-source.md | 149 ++++++++++ maintainers/adr/README.md | 1 + 11 files changed, 827 insertions(+) create mode 100644 FirstClassErrors.Testing/Any.cs create mode 100644 FirstClassErrors.UnitTests/AnyTests.cs create mode 100644 FirstClassErrors.UnitTests/ClockUseAnyTests.cs create mode 100644 FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs create mode 100644 maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md diff --git a/FirstClassErrors.Testing/Any.cs b/FirstClassErrors.Testing/Any.cs new file mode 100644 index 0000000..546865b --- /dev/null +++ b/FirstClassErrors.Testing/Any.cs @@ -0,0 +1,259 @@ +namespace FirstClassErrors.Testing; + +/// +/// Supplies arbitrary, valid values for the parts of a test that are not under assertion — the "any" a +/// test needs so its Arrange stops advertising values it never checks. Reach for it when a test says "give +/// me some error code / message / instant" and the exact value is irrelevant: an explicit +/// call reads as "this is arbitrary" where a hand-picked literal reads as "this matters". +/// +/// +/// +/// Every value is drawn from a pseudo-random source. By default that source is unseeded, so each run produces +/// fresh values; wrap the code in to make the sequence deterministic and reproducible. +/// The source is stored in an , so it flows with the current execution context and +/// never leaks across tests running in parallel — the same contract the rest of this package keeps. +/// +/// +/// The values are valid (an is never blank, an +/// is a real UTC instant) but deliberately recognizable as arbitrary (codes look like +/// ANY_CODE_7F3A9C), so an arbitrary value that surfaces in a failure message is easy to spot. +/// +/// +/// +/// // The order id is what the test asserts on; the message is not — so it is Any. +/// Outcome<Order> outcome = Outcome<Order>.Failure( +/// DomainError.Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) +/// .WithPublicMessage(Any.ShortMessage())); +/// +/// // Reproduce a run by pinning the seed: +/// using (Any.UseSeed(1234)) { +/// ErrorCode code = Any.ErrorCode(); // same value on every run +/// } +/// +/// +/// +public static class Any { + + #region Fields declarations + + private const string CodeAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + private const string TextAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789"; + + private static readonly DateTimeOffset Origin = new(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static readonly AsyncLocal Seeded = new(); + + #endregion + + /// + /// Pins the arbitrary-value source to a deterministic sequence seeded with until the + /// returned scope is disposed, so a test (or a whole run) using becomes reproducible. + /// + /// + /// Scopes nest: an inner uses its own source and restores the outer one when disposed, + /// without disturbing the outer sequence. Always scope the override with using. Outside any scope the + /// source is unseeded and every run differs. + /// + /// The seed that makes the sequence of arbitrary values reproducible. + /// A scope that restores the previous source when disposed. + public static IDisposable UseSeed(int seed) { + Random? previous = Seeded.Value; + Seeded.Value = new Random(seed); + + return new SeedScope(previous); + } + + /// + /// Returns an arbitrary drawn from the full range of the type. + /// + /// An arbitrary integer, possibly negative. + public static int Int() { + return CurrentRandom.Next(int.MinValue, int.MaxValue); + } + + /// + /// Returns an arbitrary . + /// + /// true or false, with even probability. + public static bool Bool() { + return CurrentRandom.Next(2) == 0; + } + + /// + /// Returns an arbitrary . Unlike , it is drawn from + /// the seedable source, so it is reproducible inside a scope. + /// + /// An arbitrary identifier. + public static Guid Guid() { + return NewGuid(CurrentRandom); + } + + /// + /// Returns an arbitrary UTC (offset ). + /// + /// + /// To make an error's OccurredAt arbitrary rather than a real wall-clock reading, prefer + /// Clock.UseAny(), which freezes the ambient clock to an arbitrary instant for a scope. + /// + /// An arbitrary instant, in UTC. + public static DateTimeOffset Instant() { + return NewInstant(CurrentRandom); + } + + /// + /// Returns an arbitrary, non-blank , prefixed with any- so it reads as arbitrary. + /// + /// An arbitrary string such as any-4f2a9c1b. + public static string String() { + return "any-" + NextToken(TextAlphabet, 8); + } + + /// + /// Returns an arbitrary, valid — never blank, and shaped like a real + /// code (for example ANY_CODE_7F3A9C) so it is recognizable as arbitrary. + /// + /// An arbitrary error code. + public static ErrorCode ErrorCode() { + return FirstClassErrors.ErrorCode.Create("ANY_CODE_" + NextToken(CodeAlphabet, 6)); + } + + /// + /// Returns an arbitrary, non-blank internal diagnostic message, suitable wherever a test needs a + /// diagnosticMessage it does not assert on. + /// + /// An arbitrary diagnostic message. + public static string DiagnosticMessage() { + return "Any diagnostic message " + NextToken(CodeAlphabet, 6) + "."; + } + + /// + /// Returns an arbitrary, non-blank public short message, suitable wherever a test needs a shortMessage + /// it does not assert on. + /// + /// An arbitrary short message. + public static string ShortMessage() { + return "Any short message " + NextToken(CodeAlphabet, 6) + "."; + } + + /// + /// Returns an arbitrary, non-blank public detailed message, suitable wherever a test needs a + /// detailedMessage it does not assert on. + /// + /// An arbitrary detailed message. + public static string DetailedMessage() { + return "Any detailed message " + NextToken(CodeAlphabet, 6) + "."; + } + + /// + /// Returns an arbitrary value of the enum , uniformly across all its + /// declared members. + /// + /// + /// This can return a sentinel such as Unknown. When a test needs a meaningful value, prefer the + /// dedicated helpers (, ), which exclude the + /// Unknown sentinel. + /// + /// The enum type to draw a value from. + /// An arbitrary member of . + public static TEnum Enum() + where TEnum : struct, System.Enum { + TEnum[] values = (TEnum[])System.Enum.GetValues(typeof(TEnum)); + + return values[CurrentRandom.Next(values.Length)]; + } + + /// + /// Returns an arbitrary meaningful — + /// or + /// , never + /// . + /// + /// An arbitrary transience classification other than Unknown. + public static Transience Transience() { + return NextEnumExcluding(FirstClassErrors.Transience.Unknown); + } + + /// + /// Returns an arbitrary , uniformly across all of its members. + /// + /// An arbitrary error origin. + public static ErrorOrigin ErrorOrigin() { + return Enum(); + } + + /// + /// Returns an arbitrary meaningful — + /// or + /// , never + /// . + /// + /// An arbitrary interaction direction other than Unknown. + public static InteractionDirection InteractionDirection() { + return NextEnumExcluding(FirstClassErrors.InteractionDirection.Unknown); + } + + internal static Guid NewGuid(Random random) { + byte[] bytes = new byte[16]; + random.NextBytes(bytes); + + return new Guid(bytes); + } + + internal static DateTimeOffset NewInstant(Random random) { + return Origin.AddSeconds(random.Next()); + } + + private static Random CurrentRandom { + get { + Random? random = Seeded.Value; + if (random is null) { + random = new Random(System.Guid.NewGuid().GetHashCode()); + Seeded.Value = random; + } + + return random; + } + } + + private static string NextToken(string alphabet, int length) { + Random random = CurrentRandom; + char[] chars = new char[length]; + for (int i = 0; i < length; i++) { + chars[i] = alphabet[random.Next(alphabet.Length)]; + } + + return new string(chars); + } + + private static TEnum NextEnumExcluding(params TEnum[] excluded) + where TEnum : struct, System.Enum { + List pool = new(); + foreach (TEnum value in (TEnum[])System.Enum.GetValues(typeof(TEnum))) { + if (Array.IndexOf(excluded, value) < 0) { pool.Add(value); } + } + + return pool[CurrentRandom.Next(pool.Count)]; + } + + #region Nested types + + private sealed class SeedScope : IDisposable { + + private readonly Random? _previous; + private bool _disposed; + + internal SeedScope(Random? previous) { + _previous = previous; + } + + public void Dispose() { + if (_disposed) { return; } + + _disposed = true; + Seeded.Value = _previous; + } + + } + + #endregion + +} diff --git a/FirstClassErrors.Testing/Clock.cs b/FirstClassErrors.Testing/Clock.cs index 8186d99..3e1d69b 100644 --- a/FirstClassErrors.Testing/Clock.cs +++ b/FirstClassErrors.Testing/Clock.cs @@ -48,4 +48,30 @@ public static IDisposable UseFixed(DateTimeOffset instant) { return AmbientClock.Use(() => instant); } + /// + /// Freezes the clock to an arbitrary instant for the duration of the returned scope. Use this when a test + /// needs OccurredAt to be deterministic but does not assert on the exact instant: it removes the real + /// wall-clock dependency without making you invent a literal, and reads as "the time is irrelevant here". + /// + /// + /// The instant is drawn once, when this method is called, and stays fixed for every error created within the + /// scope — the same freezing behavior as . The value comes from 's + /// unseeded source; wrap the call in Any.UseSeed(...), or use the overload, to + /// make it reproducible. + /// + /// A scope that restores the real system clock when disposed. + public static IDisposable UseAny() { + return UseFixed(Any.Instant()); + } + + /// + /// Freezes the clock to an arbitrary but reproducible instant, derived from , for + /// the duration of the returned scope. + /// + /// The seed that makes the chosen instant reproducible across runs. + /// A scope that restores the real system clock when disposed. + public static IDisposable UseAny(int seed) { + return UseFixed(Any.NewInstant(new Random(seed))); + } + } diff --git a/FirstClassErrors.Testing/InstanceIds.cs b/FirstClassErrors.Testing/InstanceIds.cs index d113f80..76dfde0 100644 --- a/FirstClassErrors.Testing/InstanceIds.cs +++ b/FirstClassErrors.Testing/InstanceIds.cs @@ -55,4 +55,31 @@ public static IDisposable UseSequential() { return Use(() => new Guid(++counter, 0, 0, new byte[8])); } + /// + /// Assigns an arbitrary identifier to each error created within the scope. Use this when a test needs + /// stable-yet-distinct ids but does not assert on their exact values: it reads as "the ids are irrelevant here". + /// + /// + /// Each error created within the scope gets its own fresh arbitrary id (as in production, several errors do not + /// collide), but the ids are drawn from 's source rather than . + /// Wrap the call in Any.UseSeed(...), or use the overload, to make the sequence + /// reproducible. To pin a single fixed id instead, use . + /// + /// A scope that restores the default (random) identifier when disposed. + public static IDisposable UseAny() { + return Use(() => Any.Guid()); + } + + /// + /// Assigns arbitrary but reproducible identifiers, drawn from a sequence seeded with + /// , to the errors created within the scope. + /// + /// The seed that makes the sequence of assigned identifiers reproducible across runs. + /// A scope that restores the default (random) identifier when disposed. + public static IDisposable UseAny(int seed) { + Random random = new(seed); + + return Use(() => Any.NewGuid(random)); + } + } diff --git a/FirstClassErrors.Testing/README.nuget.md b/FirstClassErrors.Testing/README.nuget.md index fbd732c..e3c41b8 100644 --- a/FirstClassErrors.Testing/README.nuget.md +++ b/FirstClassErrors.Testing/README.nuget.md @@ -13,6 +13,9 @@ Testing helpers for **FirstClassErrors** — so your tests about errors and outc deterministic instead of asserted over a time window. - **Freezable instance ids** (`InstanceIds.UseFixed(...)`) so `InstanceId` is stable for snapshot and equality assertions. +- **Arbitrary values** (`Any.ErrorCode()`, `Any.DiagnosticMessage()`, ...) for the + inputs a test needs but never asserts on — seedable via `Any.UseSeed(...)`, with + `Clock.UseAny()` / `InstanceIds.UseAny()` variants of the seams above. Overrides are scoped (`using`), context-local (safe under parallel tests), and never affect production behavior. diff --git a/FirstClassErrors.UnitTests/AnyTests.cs b/FirstClassErrors.UnitTests/AnyTests.cs new file mode 100644 index 0000000..69b625c --- /dev/null +++ b/FirstClassErrors.UnitTests/AnyTests.cs @@ -0,0 +1,127 @@ +#region Usings declarations + +using FirstClassErrors.Testing; + +using JetBrains.Annotations; + +using NFluent; + +#endregion + +namespace FirstClassErrors.UnitTests; + +[TestSubject(typeof(Any))] +public sealed class AnyTests { + + #region Statics members declarations + + private static string Batch() { + return string.Join("|", + Any.Int(), + Any.Bool(), + Any.Guid(), + Any.Instant().UtcTicks, + Any.String(), + Any.ErrorCode(), + Any.DiagnosticMessage(), + Any.ShortMessage(), + Any.DetailedMessage(), + Any.Transience(), + Any.ErrorOrigin(), + Any.InteractionDirection()); + } + + #endregion + + [Fact(DisplayName = "UseSeed makes the whole sequence of Any values reproducible.")] + public void UseSeedIsReproducible() { + string first; + string second; + + using (Any.UseSeed(1234)) { first = Batch(); } + using (Any.UseSeed(1234)) { second = Batch(); } + + Check.That(second).IsEqualTo(first); + } + + [Fact(DisplayName = "Different seeds produce different sequences.")] + public void DifferentSeedsDiffer() { + string fromOne; + string fromTwo; + + using (Any.UseSeed(1)) { fromOne = Batch(); } + using (Any.UseSeed(2)) { fromTwo = Batch(); } + + Check.That(fromTwo).IsNotEqualTo(fromOne); + } + + [Fact(DisplayName = "A nested UseSeed scope leaves the outer sequence undisturbed.")] + public void NestedScopeDoesNotDisturbTheOuterSequence() { + int outerFirst; + int outerSecond; + using (Any.UseSeed(7)) { + outerFirst = Any.Int(); + using (Any.UseSeed(99)) { + Any.Int(); + Any.Int(); + } + + outerSecond = Any.Int(); + } + + int expectedFirst; + int expectedSecond; + using (Any.UseSeed(7)) { + expectedFirst = Any.Int(); + expectedSecond = Any.Int(); + } + + Check.That(outerFirst).IsEqualTo(expectedFirst); + Check.That(outerSecond).IsEqualTo(expectedSecond); + } + + [Fact(DisplayName = "ErrorCode returns a non-blank code shaped like ANY_CODE_*.")] + public void ErrorCodeIsNonBlankAndRecognizable() { + string code = Any.ErrorCode().ToString(); + + Check.That(string.IsNullOrWhiteSpace(code)).IsFalse(); + Check.That(code).StartsWith("ANY_CODE_"); + } + + [Fact(DisplayName = "Instant is a UTC instant (zero offset).")] + public void InstantIsUtc() { + Check.That(Any.Instant().Offset).IsEqualTo(TimeSpan.Zero); + } + + [Fact(DisplayName = "Transience never returns the Unknown sentinel.")] + public void TransienceExcludesUnknown() { + using (Any.UseSeed(0)) { + for (int i = 0; i < 200; i++) { + Check.That(Any.Transience()).IsNotEqualTo(Transience.Unknown); + } + } + } + + [Fact(DisplayName = "InteractionDirection never returns the Unknown sentinel.")] + public void InteractionDirectionExcludesUnknown() { + using (Any.UseSeed(0)) { + for (int i = 0; i < 200; i++) { + Check.That(Any.InteractionDirection()).IsNotEqualTo(InteractionDirection.Unknown); + } + } + } + + [Fact(DisplayName = "Enum returns a defined member of the requested enum.")] + public void EnumReturnsADefinedMember() { + Check.That(System.Enum.IsDefined(typeof(Transience), Any.Enum())).IsTrue(); + } + + [Fact(DisplayName = "The string and message helpers are never blank.")] + public void TextValuesAreNeverBlank() { + Check.That(string.IsNullOrWhiteSpace(Any.String())).IsFalse(); + Check.That(string.IsNullOrWhiteSpace(Any.DiagnosticMessage())).IsFalse(); + Check.That(string.IsNullOrWhiteSpace(Any.ShortMessage())).IsFalse(); + Check.That(string.IsNullOrWhiteSpace(Any.DetailedMessage())).IsFalse(); + } + +} diff --git a/FirstClassErrors.UnitTests/ClockUseAnyTests.cs b/FirstClassErrors.UnitTests/ClockUseAnyTests.cs new file mode 100644 index 0000000..26b78ec --- /dev/null +++ b/FirstClassErrors.UnitTests/ClockUseAnyTests.cs @@ -0,0 +1,56 @@ +#region Usings declarations + +using FirstClassErrors.Testing; + +using JetBrains.Annotations; + +using NFluent; + +#endregion + +namespace FirstClassErrors.UnitTests; + +[TestSubject(typeof(Clock))] +public sealed class ClockUseAnyTests { + + #region Statics members declarations + + private static DomainError AnError() { + return DomainError.Create(ErrorCode.Create("ANY"), "diagnostic").WithPublicMessage("short"); + } + + #endregion + + [Fact(DisplayName = "Clock.UseAny freezes OccurredAt to a single arbitrary instant for the whole scope.")] + public void UseAnyFreezesASingleInstant() { + using (Clock.UseAny()) { + DomainError first = AnError(); + DomainError second = AnError(); + + Check.That(first.OccurredAt).IsEqualTo(second.OccurredAt); + } + } + + [Fact(DisplayName = "Clock.UseAny(seed) picks the same instant for a given seed.")] + public void UseAnyWithSeedIsReproducible() { + DateTimeOffset first; + DateTimeOffset second; + + using (Clock.UseAny(42)) { first = AnError().OccurredAt; } + using (Clock.UseAny(42)) { second = AnError().OccurredAt; } + + Check.That(second).IsEqualTo(first); + } + + [Fact(DisplayName = "Clock.UseAny only affects code inside the scope; the real clock resumes after disposal.")] + public void UseAnyIsScoped() { + using (Clock.UseAny(1)) { AnError(); } + + DateTimeOffset before = DateTimeOffset.UtcNow; + DateTimeOffset live = AnError().OccurredAt; + DateTimeOffset after = DateTimeOffset.UtcNow; + + Check.That(live >= before && live <= after).IsTrue(); + } + +} diff --git a/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs b/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs new file mode 100644 index 0000000..ca7d864 --- /dev/null +++ b/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs @@ -0,0 +1,69 @@ +#region Usings declarations + +using FirstClassErrors.Testing; + +using JetBrains.Annotations; + +using NFluent; + +#endregion + +namespace FirstClassErrors.UnitTests; + +[TestSubject(typeof(InstanceIds))] +public sealed class InstanceIdsUseAnyTests { + + #region Statics members declarations + + private static DomainError AnError() { + return DomainError.Create(ErrorCode.Create("ANY"), "diagnostic").WithPublicMessage("short"); + } + + #endregion + + [Fact(DisplayName = "InstanceIds.UseAny assigns a distinct arbitrary id to each error.")] + public void UseAnyAssignsDistinctIds() { + using (InstanceIds.UseAny()) { + Guid first = AnError().InstanceId; + Guid second = AnError().InstanceId; + + Check.That(first).IsNotEqualTo(second); + } + } + + [Fact(DisplayName = "InstanceIds.UseAny(seed) reproduces the same sequence of ids.")] + public void UseAnyWithSeedIsReproducible() { + Guid firstRunA; + Guid firstRunB; + Guid secondRunA; + Guid secondRunB; + + using (InstanceIds.UseAny(7)) { + firstRunA = AnError().InstanceId; + firstRunB = AnError().InstanceId; + } + + using (InstanceIds.UseAny(7)) { + secondRunA = AnError().InstanceId; + secondRunB = AnError().InstanceId; + } + + Check.That(secondRunA).IsEqualTo(firstRunA); + Check.That(secondRunB).IsEqualTo(firstRunB); + } + + [Fact(DisplayName = "InstanceIds.UseAny only affects code inside the scope.")] + public void UseAnyIsScoped() { + Guid fixedId = new("11111111-1111-1111-1111-111111111111"); + + using (InstanceIds.UseAny()) { + using (InstanceIds.UseFixed(fixedId)) { + Check.That(AnError().InstanceId).IsEqualTo(fixedId); + } + + // The inner UseFixed scope is disposed: UseAny is restored, so ids are arbitrary again. + Check.That(AnError().InstanceId).IsNotEqualTo(fixedId); + } + } + +} diff --git a/doc/DeterministicTesting.en.md b/doc/DeterministicTesting.en.md index 836d238..73840ac 100644 --- a/doc/DeterministicTesting.en.md +++ b/doc/DeterministicTesting.en.md @@ -158,6 +158,61 @@ public void A_missing_order_error_is_fully_deterministic() { } ``` +## Supply arbitrary values you don't assert on + +Freezing pins a value the test *does* care about. The mirror-image need is just as common: an input the test must provide but never checks — an error code, a message, an occurrence instant. Written as a literal it reads as if it mattered, and a constant reused across a suite can let a test pass for the wrong reason. + +`Any` supplies a valid-but-arbitrary value instead, so the one input that matters stands out and the rest announce themselves as incidental: + +```csharp +// Only the code identifies the scenario; the messages are incidental. +DomainError error = DomainError + .Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) + .WithPublicMessage(Any.ShortMessage()); + +Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); +``` + +What `Any` offers: + +| Helper | Returns | +| --- | --- | +| `Any.ErrorCode()` | a non-blank code, shaped like `ANY_CODE_7F3A9C` | +| `Any.DiagnosticMessage()` / `Any.ShortMessage()` / `Any.DetailedMessage()` | a non-blank message | +| `Any.Guid()` / `Any.Instant()` / `Any.String()` / `Any.Int()` / `Any.Bool()` | an arbitrary primitive (`Instant` is UTC) | +| `Any.Enum()` | any member of the enum (may be a sentinel such as `Unknown`) | +| `Any.Transience()` / `Any.InteractionDirection()` | a *meaningful* value — never the `Unknown` sentinel | +| `Any.ErrorOrigin()` | any `ErrorOrigin` | + +Values are recognizable as arbitrary (codes look like `ANY_CODE_…`), so an incidental value that surfaces in a failure message is easy to spot. + +### Reproduce a run with a seed + +By default the values differ on every run — which is what exposes a test that secretly depends on one. To reproduce a specific run, pin the seed for a scope; every `Any` call inside it becomes deterministic: + +```csharp +using (Any.UseSeed(1234)) { + ErrorCode code = Any.ErrorCode(); // the same value on every run +} +``` + +`Any.UseSeed(...)` follows the same scope rules as the overrides above: disposable, context-local, and nestable. + +### Arbitrary `OccurredAt` and `InstanceId` + +When a test needs stable occurrence data but does not assert the exact values, the clock and instance-id seams pair a `UseAny` with their `UseFixed`: + +```csharp +using (Clock.UseAny()) { // OccurredAt is frozen to one arbitrary instant + using (InstanceIds.UseAny()) { // each error gets its own arbitrary id + DomainError error = MakeError(); + // ... assert on the code or context, not on the time or id + } +} +``` + +Both take an optional seed (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) to make the chosen values reproducible. + ## Scope and parallel tests An override takes effect when its `using` opens and is undone when the `using` is disposed. Outside that block, the clock and instance ids are back to their real behavior. diff --git a/doc/DeterministicTesting.fr.md b/doc/DeterministicTesting.fr.md index f2081c3..35c8048 100644 --- a/doc/DeterministicTesting.fr.md +++ b/doc/DeterministicTesting.fr.md @@ -158,6 +158,61 @@ public void Une_erreur_de_commande_absente_est_totalement_deterministe() { } ``` +## Fournir des valeurs arbitraires non assertées + +Figer épingle une valeur qui compte pour le test. Le besoin symétrique est tout aussi fréquent : une entrée que le test doit fournir mais ne vérifie jamais — un code d’erreur, un message, un instant de survenue. Écrite en dur, elle se lit comme si elle comptait, et une constante réutilisée dans toute une suite peut faire passer un test pour une mauvaise raison. + +`Any` fournit à la place une valeur valide mais arbitraire : la seule entrée qui compte ressort, et les autres s’annoncent comme accessoires : + +```csharp +// Seul le code identifie le scénario ; les messages sont accessoires. +DomainError error = DomainError + .Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) + .WithPublicMessage(Any.ShortMessage()); + +Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); +``` + +Ce que `Any` propose : + +| Helper | Renvoie | +| --- | --- | +| `Any.ErrorCode()` | un code non vide, de la forme `ANY_CODE_7F3A9C` | +| `Any.DiagnosticMessage()` / `Any.ShortMessage()` / `Any.DetailedMessage()` | un message non vide | +| `Any.Guid()` / `Any.Instant()` / `Any.String()` / `Any.Int()` / `Any.Bool()` | une primitive arbitraire (`Instant` est en UTC) | +| `Any.Enum()` | un membre quelconque de l’enum (éventuellement un sentinelle comme `Unknown`) | +| `Any.Transience()` / `Any.InteractionDirection()` | une valeur *significative* — jamais le sentinelle `Unknown` | +| `Any.ErrorOrigin()` | un `ErrorOrigin` quelconque | + +Les valeurs sont reconnaissables comme arbitraires (les codes ressemblent à `ANY_CODE_…`), si bien qu’une valeur accessoire apparaissant dans un message d’échec se repère facilement. + +### Reproduire une exécution avec une graine + +Par défaut, les valeurs changent à chaque exécution — ce qui révèle justement un test qui en dépend secrètement. Pour reproduire une exécution précise, épinglez la graine sur une portée ; chaque appel à `Any` à l’intérieur devient déterministe : + +```csharp +using (Any.UseSeed(1234)) { + ErrorCode code = Any.ErrorCode(); // la même valeur à chaque exécution +} +``` + +`Any.UseSeed(...)` suit les mêmes règles de portée que les overrides ci-dessus : jetable, local au contexte, et imbriquable. + +### `OccurredAt` et `InstanceId` arbitraires + +Lorsqu’un test a besoin de données d’occurrence stables sans asserter leurs valeurs exactes, les seams de l’horloge et des identifiants proposent un `UseAny` en pendant de leur `UseFixed` : + +```csharp +using (Clock.UseAny()) { // OccurredAt est figé sur un instant arbitraire + using (InstanceIds.UseAny()) { // chaque erreur reçoit son propre identifiant arbitraire + DomainError error = MakeError(); + // ... assertez le code ou le contexte, pas l’instant ni l’identifiant + } +} +``` + +Les deux acceptent une graine optionnelle (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) pour rendre les valeurs choisies reproductibles. + ## Portée et tests parallèles Un override prend effet à l’ouverture de son `using` et est défait à sa libération. En dehors de ce bloc, l’horloge et les identifiants retrouvent leur comportement réel. diff --git a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md new file mode 100644 index 0000000..2584b71 --- /dev/null +++ b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md @@ -0,0 +1,149 @@ +# ADR-0006 | Supply arbitrary test values from a single seedable source + +**Status:** Proposed +**Date:** 2026-07-15 +**Decision Makers:** Reefact + +## Context + +The companion package `FirstClassErrors.Testing` exists so that tests about +errors and outcomes read like tests about values. It already ships two +test seams — a freezable clock and freezable instance ids — that share one +contract: overrides are scoped with `using`, disposable, and context-local +(backed by `AsyncLocal`), so they never leak across tests running in parallel. + +A test almost always needs inputs it does **not** assert on: an error code, a +diagnostic or short message, an occurrence instant, an instance id. Two facts +follow from how those inputs are supplied today. + +* **Legibility.** A hand-picked literal (`ErrorCode.Create("PAYMENT_DECLINED")`) + reads as significant — "this value matters to the test" — even when the test + never checks it. The reader cannot tell an asserted value from an incidental + one. +* **Overfitting.** A fixed constant reused across a suite can let a test pass for + the wrong reason (it happens to match that one constant). A value that varies + between runs surfaces such coupling. + +Established libraries solve this with an "anonymous / arbitrary value" facility +(AutoFixture, Bogus, the GOOS `Any` helper, FsCheck generators). Adopting one +here meets two hard constraints of this package: + +* **Zero third-party runtime dependencies.** The package's stated promise is + that it "adds nothing to your production dependencies". A shipped test-support + package taking a dependency on AutoFixture/Bogus/FsCheck would impose that + dependency, transitively, on every consumer. +* **Parallel-test safety on .NET Standard 2.0.** Test runners execute test + classes concurrently; a single shared, mutable `System.Random` is not + thread-safe and would produce cross-test interference and non-reproducible + values. `System.Random.Shared` does not exist on the netstandard2.0 target. + +Reproducing a failure that used an arbitrary value requires a **seed**: without +one, a failing run cannot be replayed. + +## Decision + +`FirstClassErrors.Testing` supplies arbitrary test values through a single, +dependency-free, context-local pseudo-random source whose determinism is opt-in +via a seed scope, and the clock and instance-id seams gain `UseAny` variants +layered on that same source. + +## Rationale + +* **Keeps the zero-dependency promise.** A first-party source built on + `System.Random` adds no package reference, so the promise that the package + adds nothing to a consumer's dependencies holds — the constraint that rules + out AutoFixture/Bogus/FsCheck. +* **Parallel-safe by reusing the package's own idiom.** Storing the source in an + `AsyncLocal` gives each execution context its own generator, which is exactly + the "never leaks across parallel tests" contract the clock and instance-id + seams already keep. The new surface is the same shape as the surface it sits + next to, rather than a second, unrelated mechanism. +* **Arbitrary by default, reproducible on demand.** An unseeded default makes + values differ between runs, which is what exposes overfitting; an opt-in seed + scope makes a chosen test or run replayable. This directly serves the + legibility and overfitting facts, and the reproducibility requirement, without + forcing every test to manage a seed. +* **The name carries the intent.** Reaching for an explicitly *arbitrary* value + reads as "this input is incidental", which is the distinction a hand-picked + literal cannot make. The `UseAny` variants extend the existing `UseFixed` / + `UseSequential` family on the same seams, so the clock and ids gain the same + "value is irrelevant here" expression the value factory offers. + +## Alternatives Considered + +### Depend on AutoFixture, Bogus, or FsCheck + +They are the mature, well-understood tools for arbitrary test data. Rejected +because each is a third-party runtime dependency that a shipped test-support +package would push onto every consumer, breaking the package's zero-dependency +promise; their broader object-graph and generator machinery also exceeds what a +small, focused "give me a value I don't assert on" helper needs. + +### A shared static source without context-locality + +The simplest implementation — one static `Random` behind the facade. Rejected +because it is not safe under parallel test execution: concurrent draws interfere, +values are not reproducible, and the coupling between unrelated tests is exactly +what the package's existing seams were designed to avoid. + +### An instance-based generator (`new …(seed)`) as the primary surface + +The AutoFixture-style shape, where a test constructs a generator and calls it. +Rejected as the primary surface because it threads generator state through each +test and diverges from the package's established `Use*` / disposable-scope idiom; +it remains available as a possible future addition for callers who want an +explicit object, but it is not the shape the package leads with. + +### Add test factories to the value objects themselves + +Put "make an arbitrary one" on `ErrorCode`, `Error`, and friends in the core +library. Rejected because it mixes a test concern into production types and would +ship test-only surface in the main package. + +## Consequences + +### Positive + +* An incidental input is legible as incidental at the call site, and a failing + run is reproducible once a seed is set. +* No new dependency reaches consumers, and the new surface reuses the package's + existing parallel-safe, disposable-scope contract rather than inventing a + second one. + +### Negative + +* A new public surface on a shipped package that must be maintained and kept + documented in English and French. +* The generic enum helper can still return a sentinel value (such as `Unknown`); + callers that need a *meaningful* value must use the dedicated per-enum helpers + that exclude it. +* `ErrorContextKey` is deliberately left out of the first surface: keys live in a + process-wide registry with no public reset, so minting arbitrary keys would + accumulate global state across a run. Tests needing an arbitrary key are + unserved until that is designed. + +### Risks + +* The unseeded default draws from a per-context generator; if two contexts + coincidentally start from the same seed, their "arbitrary" values coincide. + This is harmless (the values are not asserted on) and is mitigated by deriving + the default seed from a fresh identifier per context. +* Reproducing a failure still depends on the author having set a seed; an + unseeded failing test is not replayable from its output alone. Surfacing the + auto-chosen seed for replay is a possible later refinement. +* Until enforced by tooling, the "arbitrary value ⇒ use the source, not a + literal" habit holds only through review and documentation. + +## Follow-up Actions + +* Document the surface in the testing guide, in English and French, in lockstep. +* Consider a design for arbitrary `ErrorContextKey` values that respects the + process-wide registry. +* Consider surfacing the auto-chosen seed so an unseeded failure can be replayed. +* Consider an instance-based generator if callers ask for an explicit object. + +## References + +* `doc/DeterministicTesting.en.md` — the guide where the new surface is documented. +* ADR-0005 — prior naming decision in the same spirit (a name should announce + what a call does); context only, not a precedent for this choice. diff --git a/maintainers/adr/README.md b/maintainers/adr/README.md index d9a300f..b549314 100644 --- a/maintainers/adr/README.md +++ b/maintainers/adr/README.md @@ -174,3 +174,4 @@ Optional supporting material: | [ADR-0003](0003-unify-outcome-value-mapping-under-then.md) | Unify Outcome value mapping under Then | Accepted | | [ADR-0004](0004-check-every-pull-request-against-the-adr-base.md) | Check every pull request against the ADR base | Accepted | | [ADR-0005](0005-reserve-the-plain-factory-name-for-the-outcome-returning-variant.md) | Reserve the plain factory name for the Outcome-returning variant | Proposed | +| [ADR-0006](0006-supply-arbitrary-test-values-from-a-seedable-source.md) | Supply arbitrary test values from a single seedable source | Proposed | From d4932181331b8633df4e302adcb3bc773aa8f2ab Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 21:36:36 +0000 Subject: [PATCH 2/4] refactor(testing): isolate the generic value engine behind Any MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the seedable, context-local random engine and its primitive generators into an internal `ArbitrarySource`, leaving `Any` as a thin, error-aware facade over it. The clock and instance-id seams now draw their arbitrary values from the same engine. The public surface of `Any` (and of `Clock.UseAny` / `InstanceIds.UseAny`) is unchanged — this is a pure reorganization. Isolating the generic engine from the error-specific helpers keeps a future extraction into a standalone, error-agnostic utility a mechanical move rather than a rewrite; ADR-0006 records that as a follow-up trigger. Refs: #135 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012ULkMmGS2EgnALAJtwpxtP --- FirstClassErrors.Testing/Any.cs | 109 +++------------- FirstClassErrors.Testing/ArbitrarySource.cs | 119 ++++++++++++++++++ FirstClassErrors.Testing/Clock.cs | 4 +- FirstClassErrors.Testing/InstanceIds.cs | 4 +- ...rary-test-values-from-a-seedable-source.md | 3 + 5 files changed, 146 insertions(+), 93 deletions(-) create mode 100644 FirstClassErrors.Testing/ArbitrarySource.cs diff --git a/FirstClassErrors.Testing/Any.cs b/FirstClassErrors.Testing/Any.cs index 546865b..50ea1fd 100644 --- a/FirstClassErrors.Testing/Any.cs +++ b/FirstClassErrors.Testing/Any.cs @@ -10,14 +10,18 @@ namespace FirstClassErrors.Testing; /// /// Every value is drawn from a pseudo-random source. By default that source is unseeded, so each run produces /// fresh values; wrap the code in to make the sequence deterministic and reproducible. -/// The source is stored in an , so it flows with the current execution context and -/// never leaks across tests running in parallel — the same contract the rest of this package keeps. +/// The source flows with the current execution context, so it never leaks across tests running in parallel — +/// the same contract the rest of this package keeps. /// /// /// The values are valid (an is never blank, an /// is a real UTC instant) but deliberately recognizable as arbitrary (codes look like /// ANY_CODE_7F3A9C), so an arbitrary value that surfaces in a failure message is easy to spot. /// +/// +/// is the error-aware surface; the generic value engine it delegates to is internal, so a +/// test never depends on it directly. +/// /// /// /// // The order id is what the test asserts on; the message is not — so it is Any. @@ -39,9 +43,6 @@ public static class Any { private const string CodeAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; private const string TextAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789"; - private static readonly DateTimeOffset Origin = new(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); - private static readonly AsyncLocal Seeded = new(); - #endregion /// @@ -56,10 +57,7 @@ public static class Any { /// The seed that makes the sequence of arbitrary values reproducible. /// A scope that restores the previous source when disposed. public static IDisposable UseSeed(int seed) { - Random? previous = Seeded.Value; - Seeded.Value = new Random(seed); - - return new SeedScope(previous); + return ArbitrarySource.UseSeed(seed); } /// @@ -67,7 +65,7 @@ public static IDisposable UseSeed(int seed) { /// /// An arbitrary integer, possibly negative. public static int Int() { - return CurrentRandom.Next(int.MinValue, int.MaxValue); + return ArbitrarySource.Int(); } /// @@ -75,7 +73,7 @@ public static int Int() { /// /// true or false, with even probability. public static bool Bool() { - return CurrentRandom.Next(2) == 0; + return ArbitrarySource.Bool(); } /// @@ -84,7 +82,7 @@ public static bool Bool() { /// /// An arbitrary identifier. public static Guid Guid() { - return NewGuid(CurrentRandom); + return ArbitrarySource.Guid(); } /// @@ -96,7 +94,7 @@ public static Guid Guid() { /// /// An arbitrary instant, in UTC. public static DateTimeOffset Instant() { - return NewInstant(CurrentRandom); + return ArbitrarySource.Instant(); } /// @@ -104,7 +102,7 @@ public static DateTimeOffset Instant() { /// /// An arbitrary string such as any-4f2a9c1b. public static string String() { - return "any-" + NextToken(TextAlphabet, 8); + return "any-" + ArbitrarySource.Token(TextAlphabet, 8); } /// @@ -113,7 +111,7 @@ public static string String() { /// /// An arbitrary error code. public static ErrorCode ErrorCode() { - return FirstClassErrors.ErrorCode.Create("ANY_CODE_" + NextToken(CodeAlphabet, 6)); + return FirstClassErrors.ErrorCode.Create("ANY_CODE_" + ArbitrarySource.Token(CodeAlphabet, 6)); } /// @@ -122,7 +120,7 @@ public static ErrorCode ErrorCode() { /// /// An arbitrary diagnostic message. public static string DiagnosticMessage() { - return "Any diagnostic message " + NextToken(CodeAlphabet, 6) + "."; + return "Any diagnostic message " + ArbitrarySource.Token(CodeAlphabet, 6) + "."; } /// @@ -131,7 +129,7 @@ public static string DiagnosticMessage() { /// /// An arbitrary short message. public static string ShortMessage() { - return "Any short message " + NextToken(CodeAlphabet, 6) + "."; + return "Any short message " + ArbitrarySource.Token(CodeAlphabet, 6) + "."; } /// @@ -140,7 +138,7 @@ public static string ShortMessage() { /// /// An arbitrary detailed message. public static string DetailedMessage() { - return "Any detailed message " + NextToken(CodeAlphabet, 6) + "."; + return "Any detailed message " + ArbitrarySource.Token(CodeAlphabet, 6) + "."; } /// @@ -156,9 +154,7 @@ public static string DetailedMessage() { /// An arbitrary member of . public static TEnum Enum() where TEnum : struct, System.Enum { - TEnum[] values = (TEnum[])System.Enum.GetValues(typeof(TEnum)); - - return values[CurrentRandom.Next(values.Length)]; + return ArbitrarySource.Enum(); } /// @@ -169,7 +165,7 @@ public static TEnum Enum() /// /// An arbitrary transience classification other than Unknown. public static Transience Transience() { - return NextEnumExcluding(FirstClassErrors.Transience.Unknown); + return ArbitrarySource.EnumExcluding(FirstClassErrors.Transience.Unknown); } /// @@ -177,7 +173,7 @@ public static Transience Transience() { /// /// An arbitrary error origin. public static ErrorOrigin ErrorOrigin() { - return Enum(); + return ArbitrarySource.Enum(); } /// @@ -188,72 +184,7 @@ public static ErrorOrigin ErrorOrigin() { /// /// An arbitrary interaction direction other than Unknown. public static InteractionDirection InteractionDirection() { - return NextEnumExcluding(FirstClassErrors.InteractionDirection.Unknown); - } - - internal static Guid NewGuid(Random random) { - byte[] bytes = new byte[16]; - random.NextBytes(bytes); - - return new Guid(bytes); - } - - internal static DateTimeOffset NewInstant(Random random) { - return Origin.AddSeconds(random.Next()); - } - - private static Random CurrentRandom { - get { - Random? random = Seeded.Value; - if (random is null) { - random = new Random(System.Guid.NewGuid().GetHashCode()); - Seeded.Value = random; - } - - return random; - } - } - - private static string NextToken(string alphabet, int length) { - Random random = CurrentRandom; - char[] chars = new char[length]; - for (int i = 0; i < length; i++) { - chars[i] = alphabet[random.Next(alphabet.Length)]; - } - - return new string(chars); - } - - private static TEnum NextEnumExcluding(params TEnum[] excluded) - where TEnum : struct, System.Enum { - List pool = new(); - foreach (TEnum value in (TEnum[])System.Enum.GetValues(typeof(TEnum))) { - if (Array.IndexOf(excluded, value) < 0) { pool.Add(value); } - } - - return pool[CurrentRandom.Next(pool.Count)]; + return ArbitrarySource.EnumExcluding(FirstClassErrors.InteractionDirection.Unknown); } - #region Nested types - - private sealed class SeedScope : IDisposable { - - private readonly Random? _previous; - private bool _disposed; - - internal SeedScope(Random? previous) { - _previous = previous; - } - - public void Dispose() { - if (_disposed) { return; } - - _disposed = true; - Seeded.Value = _previous; - } - - } - - #endregion - } diff --git a/FirstClassErrors.Testing/ArbitrarySource.cs b/FirstClassErrors.Testing/ArbitrarySource.cs new file mode 100644 index 0000000..cad4622 --- /dev/null +++ b/FirstClassErrors.Testing/ArbitrarySource.cs @@ -0,0 +1,119 @@ +namespace FirstClassErrors.Testing; + +/// +/// The generic, error-agnostic engine behind : a seedable, context-local source of arbitrary +/// primitive values (numbers, booleans, identifiers, instants, tokens, enum members). It carries no knowledge of +/// FirstClassErrors — layers the error-specific helpers on top, and the clock and instance-id +/// seams draw their arbitrary values from here — so it stays a self-contained unit that could be extracted as a +/// standalone utility unchanged. +/// +/// +/// The source is stored in an , so it flows with the current execution context and +/// never leaks across tests running in parallel. Outside a scope it is unseeded and every +/// run differs; inside one it is deterministic. +/// +internal static class ArbitrarySource { + + #region Fields declarations + + private static readonly DateTimeOffset Origin = new(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static readonly AsyncLocal Seeded = new(); + + #endregion + + internal static Random Current { + get { + Random? random = Seeded.Value; + if (random is null) { + random = new Random(System.Guid.NewGuid().GetHashCode()); + Seeded.Value = random; + } + + return random; + } + } + + internal static IDisposable UseSeed(int seed) { + Random? previous = Seeded.Value; + Seeded.Value = new Random(seed); + + return new SeedScope(previous); + } + + internal static int Int() { + return Current.Next(int.MinValue, int.MaxValue); + } + + internal static bool Bool() { + return Current.Next(2) == 0; + } + + internal static Guid Guid() { + return NewGuid(Current); + } + + internal static DateTimeOffset Instant() { + return NewInstant(Current); + } + + internal static string Token(string alphabet, int length) { + Random random = Current; + char[] chars = new char[length]; + for (int i = 0; i < length; i++) { + chars[i] = alphabet[random.Next(alphabet.Length)]; + } + + return new string(chars); + } + + internal static TEnum Enum() + where TEnum : struct, System.Enum { + TEnum[] values = (TEnum[])System.Enum.GetValues(typeof(TEnum)); + + return values[Current.Next(values.Length)]; + } + + internal static TEnum EnumExcluding(params TEnum[] excluded) + where TEnum : struct, System.Enum { + List pool = new(); + foreach (TEnum value in (TEnum[])System.Enum.GetValues(typeof(TEnum))) { + if (Array.IndexOf(excluded, value) < 0) { pool.Add(value); } + } + + return pool[Current.Next(pool.Count)]; + } + + internal static Guid NewGuid(Random random) { + byte[] bytes = new byte[16]; + random.NextBytes(bytes); + + return new Guid(bytes); + } + + internal static DateTimeOffset NewInstant(Random random) { + return Origin.AddSeconds(random.Next()); + } + + #region Nested types + + private sealed class SeedScope : IDisposable { + + private readonly Random? _previous; + private bool _disposed; + + internal SeedScope(Random? previous) { + _previous = previous; + } + + public void Dispose() { + if (_disposed) { return; } + + _disposed = true; + Seeded.Value = _previous; + } + + } + + #endregion + +} diff --git a/FirstClassErrors.Testing/Clock.cs b/FirstClassErrors.Testing/Clock.cs index 3e1d69b..a8aa3b4 100644 --- a/FirstClassErrors.Testing/Clock.cs +++ b/FirstClassErrors.Testing/Clock.cs @@ -61,7 +61,7 @@ public static IDisposable UseFixed(DateTimeOffset instant) { /// /// A scope that restores the real system clock when disposed. public static IDisposable UseAny() { - return UseFixed(Any.Instant()); + return UseFixed(ArbitrarySource.Instant()); } /// @@ -71,7 +71,7 @@ public static IDisposable UseAny() { /// The seed that makes the chosen instant reproducible across runs. /// A scope that restores the real system clock when disposed. public static IDisposable UseAny(int seed) { - return UseFixed(Any.NewInstant(new Random(seed))); + return UseFixed(ArbitrarySource.NewInstant(new Random(seed))); } } diff --git a/FirstClassErrors.Testing/InstanceIds.cs b/FirstClassErrors.Testing/InstanceIds.cs index 76dfde0..691fd18 100644 --- a/FirstClassErrors.Testing/InstanceIds.cs +++ b/FirstClassErrors.Testing/InstanceIds.cs @@ -67,7 +67,7 @@ public static IDisposable UseSequential() { /// /// A scope that restores the default (random) identifier when disposed. public static IDisposable UseAny() { - return Use(() => Any.Guid()); + return Use(() => ArbitrarySource.Guid()); } /// @@ -79,7 +79,7 @@ public static IDisposable UseAny() { public static IDisposable UseAny(int seed) { Random random = new(seed); - return Use(() => Any.NewGuid(random)); + return Use(() => ArbitrarySource.NewGuid(random)); } } diff --git a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md index 2584b71..07c0d8b 100644 --- a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md +++ b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md @@ -141,6 +141,9 @@ ship test-only surface in the main package. process-wide registry. * Consider surfacing the auto-chosen seed so an unseeded failure can be replayed. * Consider an instance-based generator if callers ask for an explicit object. +* Consider extracting the generic value engine into a standalone, error-agnostic + utility if a second consumer appears; it is kept internally separable from the + error-specific surface to that end. ## References From a474e902f1ab571e7e4c89701cc4315e9a7a5c28 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 22:18:55 +0000 Subject: [PATCH 3/4] docs(testing): give arbitrary test values a dedicated guide The testing guide was split upstream: assertions in Testing.*.md and the clock/instance-id determinism seams in DeterministicTesting.*.md. The Any factory straddles both, so give it its own page rather than wedging it into either. - Add doc/ArbitraryTestValues.en.md and its French translation, covering the Any value helpers, Any.UseSeed, and the Clock.UseAny / InstanceIds.UseAny seams. - Replace the interim Any section in DeterministicTesting.*.md with a short cross-reference to the new page. - Insert the page into the navigation chain (Deterministic Error Tests -> Arbitrary Test Values -> Operational Integration) and nest both testing sub-pages under the Testing Guide in the README table of contents, English and French in lockstep. - Point the ADR-0006 reference at the new page. Refs: #135 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012ULkMmGS2EgnALAJtwpxtP --- README.md | 2 + doc/ArbitraryTestValues.en.md | 110 ++++++++++++++++++ doc/ArbitraryTestValues.fr.md | 110 ++++++++++++++++++ doc/DeterministicTesting.en.md | 57 +-------- doc/DeterministicTesting.fr.md | 57 +-------- doc/OperationalIntegration.en.md | 2 +- doc/OperationalIntegration.fr.md | 2 +- doc/README.fr.md | 2 + ...rary-test-values-from-a-seedable-source.md | 2 +- 9 files changed, 233 insertions(+), 111 deletions(-) create mode 100644 doc/ArbitraryTestValues.en.md create mode 100644 doc/ArbitraryTestValues.fr.md diff --git a/README.md b/README.md index b075b80..81ba0ef 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,8 @@ For security vulnerabilities, follow the private process in [SECURITY.md](SECURI - [Usage Patterns](doc/UsagePatterns.en.md) - [Best Practices](doc/BestPractices.en.md) - [Testing Guide](doc/Testing.en.md) + - [Deterministic Error Tests](doc/DeterministicTesting.en.md) + - [Arbitrary Test Values](doc/ArbitraryTestValues.en.md) ### Generate and operate the catalog diff --git a/doc/ArbitraryTestValues.en.md b/doc/ArbitraryTestValues.en.md new file mode 100644 index 0000000..bc85438 --- /dev/null +++ b/doc/ArbitraryTestValues.en.md @@ -0,0 +1,110 @@ +# Arbitrary Test Values + +🌍 **Languages:** +🇬🇧 English (this file) | 🇫🇷 [Français](./ArbitraryTestValues.fr.md) + +A large part of a test's `Arrange` is usually values the test never checks — an error code, a diagnostic message, an occurrence instant. Spelled out as literals they read as if they mattered, and a constant reused across a suite can let a test pass for the wrong reason. `Any` supplies a valid-but-arbitrary value instead, so the one input that matters stands out and the rest announce themselves as incidental. + +`Any` lives in **`FirstClassErrors.Testing`**; it adds no dependency and, like the clock and instance-id overrides, is scoped, context-local, and safe under parallel tests. For freezing values a test *does* assert on, see [Deterministic Error Tests](DeterministicTesting.en.md). + +## Supply an arbitrary value + +Compare a test that hard-codes every input with one that keeps only the value under assertion explicit: + +```csharp +// 😐 Before — which of these values does the test actually check? +DomainError error = DomainError + .Create(ErrorCode.Create("ORDER_NOT_FOUND"), "Order 42 was not found.") + .WithPublicMessage("The order does not exist."); + +Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); +``` + +```csharp +// 🙂 After — the code is the subject; the messages are Any. +DomainError error = DomainError + .Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) + .WithPublicMessage(Any.ShortMessage()); + +Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); +``` + +A value is only incidental when it cannot steer the code under test. If it feeds a branch, a validation, a serialization, or an ordering, it shapes the behavior even though the test never asserts it — and it cannot safely be left arbitrary. Reach for `Any` for inputs the test carries but does not act on. + +## What `Any` offers + +Every helper returns a value that is **valid for its type** — non-blank strings and messages, a real UTC instant, an error code that is never blank: + +| Helper | Returns | +| --- | --- | +| `Any.ErrorCode()` | a valid, non-blank code of the form `ANY_CODE_` + 6 uppercase alphanumerics | +| `Any.DiagnosticMessage()` / `Any.ShortMessage()` / `Any.DetailedMessage()` | a non-blank message of bounded length | +| `Any.String()` | a non-empty string of bounded length (`any-` + 8 lowercase alphanumerics); no whitespace | +| `Any.Guid()` | an arbitrary `Guid` | +| `Any.Instant()` | an arbitrary UTC instant (offset zero) between 1 January 2000 and around 2068 | +| `Any.Int()` | an arbitrary `int` — it may be negative or zero | +| `Any.Bool()` | `true` or `false` | +| `Any.Enum()` | any member of the enum — a sentinel such as `Unknown` included | +| `Any.Transience()` / `Any.InteractionDirection()` | a *meaningful* value — never the `Unknown` sentinel | +| `Any.ErrorOrigin()` | any `ErrorOrigin`; all three values are meaningful, so there is no sentinel to exclude | + +The guarantees stop at type validity. A helper does not target a domain precondition — `Any.Int()` may be negative, `Any.String()` is not a well-formed email — so a value object with a stricter contract needs its own arbitrary factory rather than a raw primitive. + +Use `Any.Enum()` when any member will do — a sentinel included — and the named enum helpers when the test needs a value that actually drives the behavior under test. + +## Reproduce a run with a seed + +The source is unseeded by default, so the values differ between runs. That is deliberate: a test that passes only for one particular value is relying on something it never states, and varying the value surfaces that coupling. + +The cost is reproducibility. The default seed is **not surfaced today**, so a run that failed on a particular unseeded value cannot be replayed from its output alone. Keep the arbitrary value out of what decides pass or fail — or, when a specific run must be reproducible, pin a seed on the narrowest useful scope; a suite-wide seed is fine when full stability is preferable to variation between runs. Every `Any` call inside the scope then becomes deterministic: + +```csharp +using (Any.UseSeed(1234)) { + ErrorCode first = Any.ErrorCode(); + ErrorCode second = Any.ErrorCode(); // the same two values on every run +} +``` + +Seeds nest: an inner `Any.UseSeed(...)` uses its own sequence and restores the outer one when it exits. Outside any scope the source is unseeded and every run differs. + +## Arbitrary `OccurredAt` and `InstanceId` + +Occurrence data is arbitrary in the same sense: a test often needs it stable without asserting the exact instant or id. The clock and instance-id seams therefore pair a `UseAny` with their `UseFixed`. `Clock.UseAny()` freezes a single arbitrary instant for the scope, while `InstanceIds.UseAny()` hands each error its own distinct arbitrary id: + +```csharp +DomainError NewError() => + DomainError.Create(Any.ErrorCode(), Any.DiagnosticMessage()).WithPublicMessage(Any.ShortMessage()); + +using (Clock.UseAny()) +using (InstanceIds.UseAny()) { + DomainError first = NewError(); + DomainError second = NewError(); + + Check.That(second.OccurredAt).IsEqualTo(first.OccurredAt); // one arbitrary instant, shared + Check.That(second.InstanceId).IsNotEqualTo(first.InstanceId); // distinct arbitrary ids +} +``` + +Both take an optional seed (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) to make the chosen values reproducible. To pin a *specific* instant or id instead, use `UseFixed` — see [Deterministic Error Tests](DeterministicTesting.en.md). + +## Scope and parallel tests + +`Any.UseSeed`, `Clock.UseAny`, and `InstanceIds.UseAny` all take effect only inside their `using` block and are restored when the scope exits. The seeded source is stored in an `AsyncLocal`, so it follows the test's own execution flow and never leaks into other tests running at the same time. + +## Review checklist + +Before reaching for an arbitrary value, verify that: + +- the value does **not** change the functional path the test exercises — it must not feed a branch, a validation, a serialization, or an ordering, even indirectly; +- the value is genuinely not checked by the test — otherwise use a literal; +- a named enum helper is used when the test needs a meaningful value, rather than `Any.Enum()`; +- a seed is pinned whenever a failing run would otherwise be irreproducible; +- `Clock.UseAny` / `InstanceIds.UseAny` are used for stable-but-irrelevant occurrence data, and `UseFixed` when the exact value is asserted. + +--- + + + +--- diff --git a/doc/ArbitraryTestValues.fr.md b/doc/ArbitraryTestValues.fr.md new file mode 100644 index 0000000..d2a4d91 --- /dev/null +++ b/doc/ArbitraryTestValues.fr.md @@ -0,0 +1,110 @@ +# Valeurs de test arbitraires + +🌍 **Langues :** +🇫🇷 Français (ce fichier) | 🇬🇧 [English](./ArbitraryTestValues.en.md) + +Une grande partie de l’`Arrange` d’un test est d’ordinaire faite de valeurs qu’il ne vérifie jamais — un code d’erreur, un message de diagnostic, un instant de survenue. Écrites en dur, elles se lisent comme si elles comptaient, et une constante réutilisée dans toute une suite peut faire passer un test pour une mauvaise raison. `Any` fournit à la place une valeur valide mais arbitraire : la seule entrée qui compte ressort, et les autres s’annoncent comme accessoires. + +`Any` vit dans **`FirstClassErrors.Testing`** ; il n’ajoute aucune dépendance et, comme les overrides d’horloge et d’identifiants, il est borné, local au contexte et sûr en tests parallèles. Pour figer les valeurs qu’un test *assertit*, voir [Tests d’erreur déterministes](DeterministicTesting.fr.md). + +## Fournir une valeur arbitraire + +Comparez un test qui code en dur chaque entrée à un test qui ne garde explicite que la valeur assertée : + +```csharp +// 😐 Avant — laquelle de ces valeurs le test vérifie-t-il réellement ? +DomainError error = DomainError + .Create(ErrorCode.Create("ORDER_NOT_FOUND"), "La commande 42 est introuvable.") + .WithPublicMessage("La commande n’existe pas."); + +Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); +``` + +```csharp +// 🙂 Après — le code est le sujet ; les messages sont fournis par Any. +DomainError error = DomainError + .Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) + .WithPublicMessage(Any.ShortMessage()); + +Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); +``` + +Une valeur n’est accessoire que si elle ne peut pas orienter le code testé. Si elle alimente une branche, une validation, une sérialisation ou un classement, elle façonne le comportement même si le test ne l’assertit jamais — et elle ne peut alors pas être laissée arbitraire sans risque. Réservez `Any` aux entrées que le test transporte mais sur lesquelles il n’agit pas. + +## Ce que `Any` propose + +Chaque helper renvoie une valeur **valide pour son type** — des chaînes et messages non vides, un instant UTC réel, un code d’erreur jamais vide : + +| Helper | Renvoie | +| --- | --- | +| `Any.ErrorCode()` | un code valide non vide, de la forme `ANY_CODE_` + 6 caractères alphanumériques majuscules | +| `Any.DiagnosticMessage()` / `Any.ShortMessage()` / `Any.DetailedMessage()` | un message non vide, de longueur bornée | +| `Any.String()` | une chaîne non vide de longueur bornée (`any-` + 8 caractères alphanumériques minuscules) ; sans espace | +| `Any.Guid()` | un `Guid` arbitraire | +| `Any.Instant()` | un instant UTC arbitraire (offset zéro), entre le 1er janvier 2000 et environ 2068 | +| `Any.Int()` | un `int` arbitraire — éventuellement négatif ou nul | +| `Any.Bool()` | `true` ou `false` | +| `Any.Enum()` | un membre quelconque de l’enum — une sentinelle comme `Unknown` comprise | +| `Any.Transience()` / `Any.InteractionDirection()` | une valeur *significative* — jamais la sentinelle `Unknown` | +| `Any.ErrorOrigin()` | un `ErrorOrigin` quelconque ; les trois valeurs sont significatives, il n’y a donc pas de sentinelle à exclure | + +Les garanties s’arrêtent à la validité du type. Un helper ne vise aucune précondition métier — `Any.Int()` peut être négatif, `Any.String()` n’est pas un e-mail bien formé — donc un value object au contrat plus strict a besoin de sa propre factory arbitraire, pas d’une primitive brute. + +Utilisez `Any.Enum()` quand n’importe quel membre convient — sentinelle comprise — et les helpers d’enum nommés quand le test a besoin d’une valeur qui déclenche réellement le comportement concerné. + +## Reproduire une exécution avec une graine + +La source n’est pas seedée par défaut : les valeurs diffèrent donc d’une exécution à l’autre. C’est délibéré : un test qui ne passe que pour une valeur particulière dépend de quelque chose qu’il n’énonce pas, et faire varier la valeur révèle ce couplage. + +Le coût, c’est la reproductibilité. La graine par défaut **n’est pas exposée aujourd’hui** : une exécution ayant échoué sur une valeur non seedée précise ne peut donc pas être rejouée à partir de sa seule sortie. Gardez la valeur arbitraire hors de ce qui décide du succès ou de l’échec — ou, lorsqu’une exécution précise doit être reproductible, épinglez une graine sur la portée la plus étroite utile ; une graine commune à toute la suite convient lorsque la stabilité complète est préférable à la variation entre les exécutions. Chaque appel à `Any` dans la portée devient alors déterministe : + +```csharp +using (Any.UseSeed(1234)) { + ErrorCode first = Any.ErrorCode(); + ErrorCode second = Any.ErrorCode(); // les deux mêmes valeurs à chaque exécution +} +``` + +Les graines s’imbriquent : un `Any.UseSeed(...)` interne utilise sa propre séquence et restaure celle de l’extérieur à sa sortie. Hors de toute portée, la source n’est pas seedée et chaque exécution diffère. + +## `OccurredAt` et `InstanceId` arbitraires + +Les données d’occurrence sont arbitraires au même sens : un test a souvent besoin qu’elles soient stables sans en vérifier l’instant ou l’identifiant exact. Les seams de l’horloge et des identifiants proposent donc un `UseAny` en pendant de leur `UseFixed`. `Clock.UseAny()` fige un unique instant arbitraire pour la portée, tandis que `InstanceIds.UseAny()` attribue à chaque erreur son propre identifiant arbitraire distinct : + +```csharp +DomainError NewError() => + DomainError.Create(Any.ErrorCode(), Any.DiagnosticMessage()).WithPublicMessage(Any.ShortMessage()); + +using (Clock.UseAny()) +using (InstanceIds.UseAny()) { + DomainError first = NewError(); + DomainError second = NewError(); + + Check.That(second.OccurredAt).IsEqualTo(first.OccurredAt); // un instant arbitraire, partagé + Check.That(second.InstanceId).IsNotEqualTo(first.InstanceId); // des identifiants arbitraires distincts +} +``` + +Les deux acceptent une graine optionnelle (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) pour rendre les valeurs choisies reproductibles. Pour épingler un instant ou un identifiant *précis*, utilisez `UseFixed` — voir [Tests d’erreur déterministes](DeterministicTesting.fr.md). + +## Portée et tests parallèles + +`Any.UseSeed`, `Clock.UseAny` et `InstanceIds.UseAny` ne prennent effet qu’à l’intérieur de leur bloc `using` et sont restaurés à la sortie de la portée. La source seedée est stockée dans un `AsyncLocal` : elle suit le flux d’exécution du test lui-même et ne fuit jamais dans d’autres tests s’exécutant en même temps. + +## Checklist de revue + +Avant de recourir à une valeur arbitraire, vérifiez que : + +- la valeur ne **modifie pas** le chemin fonctionnel exercé par le test — elle ne doit alimenter ni une branche, ni une validation, ni une sérialisation, ni un classement, même indirectement ; +- la valeur n’est réellement pas vérifiée par le test — sinon utilisez un littéral ; +- un helper d’enum nommé est utilisé quand le test a besoin d’une valeur significative, plutôt que `Any.Enum()` ; +- une graine est épinglée dès qu’une exécution en échec serait sinon irreproductible ; +- `Clock.UseAny` / `InstanceIds.UseAny` servent pour des données d’occurrence stables mais sans importance, et `UseFixed` lorsque la valeur exacte est assertée. + +--- + + + +--- diff --git a/doc/DeterministicTesting.en.md b/doc/DeterministicTesting.en.md index 73840ac..d1af274 100644 --- a/doc/DeterministicTesting.en.md +++ b/doc/DeterministicTesting.en.md @@ -158,60 +158,9 @@ public void A_missing_order_error_is_fully_deterministic() { } ``` -## Supply arbitrary values you don't assert on +## Arbitrary occurrence data -Freezing pins a value the test *does* care about. The mirror-image need is just as common: an input the test must provide but never checks — an error code, a message, an occurrence instant. Written as a literal it reads as if it mattered, and a constant reused across a suite can let a test pass for the wrong reason. - -`Any` supplies a valid-but-arbitrary value instead, so the one input that matters stands out and the rest announce themselves as incidental: - -```csharp -// Only the code identifies the scenario; the messages are incidental. -DomainError error = DomainError - .Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) - .WithPublicMessage(Any.ShortMessage()); - -Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); -``` - -What `Any` offers: - -| Helper | Returns | -| --- | --- | -| `Any.ErrorCode()` | a non-blank code, shaped like `ANY_CODE_7F3A9C` | -| `Any.DiagnosticMessage()` / `Any.ShortMessage()` / `Any.DetailedMessage()` | a non-blank message | -| `Any.Guid()` / `Any.Instant()` / `Any.String()` / `Any.Int()` / `Any.Bool()` | an arbitrary primitive (`Instant` is UTC) | -| `Any.Enum()` | any member of the enum (may be a sentinel such as `Unknown`) | -| `Any.Transience()` / `Any.InteractionDirection()` | a *meaningful* value — never the `Unknown` sentinel | -| `Any.ErrorOrigin()` | any `ErrorOrigin` | - -Values are recognizable as arbitrary (codes look like `ANY_CODE_…`), so an incidental value that surfaces in a failure message is easy to spot. - -### Reproduce a run with a seed - -By default the values differ on every run — which is what exposes a test that secretly depends on one. To reproduce a specific run, pin the seed for a scope; every `Any` call inside it becomes deterministic: - -```csharp -using (Any.UseSeed(1234)) { - ErrorCode code = Any.ErrorCode(); // the same value on every run -} -``` - -`Any.UseSeed(...)` follows the same scope rules as the overrides above: disposable, context-local, and nestable. - -### Arbitrary `OccurredAt` and `InstanceId` - -When a test needs stable occurrence data but does not assert the exact values, the clock and instance-id seams pair a `UseAny` with their `UseFixed`: - -```csharp -using (Clock.UseAny()) { // OccurredAt is frozen to one arbitrary instant - using (InstanceIds.UseAny()) { // each error gets its own arbitrary id - DomainError error = MakeError(); - // ... assert on the code or context, not on the time or id - } -} -``` - -Both take an optional seed (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) to make the chosen values reproducible. +When a test needs `OccurredAt` and `InstanceId` to be stable but does not assert their exact values, freeze them to an *arbitrary* value with `Clock.UseAny()` and `InstanceIds.UseAny()` rather than `UseFixed`; both take an optional seed for reproducibility. They belong to the broader `Any` factory for values a test does not assert on — see [Arbitrary Test Values](ArbitraryTestValues.en.md). ## Scope and parallel tests @@ -254,7 +203,7 @@ Before approving a deterministic error test, verify that: --- --- \ No newline at end of file diff --git a/doc/DeterministicTesting.fr.md b/doc/DeterministicTesting.fr.md index 35c8048..c43f78b 100644 --- a/doc/DeterministicTesting.fr.md +++ b/doc/DeterministicTesting.fr.md @@ -158,60 +158,9 @@ public void Une_erreur_de_commande_absente_est_totalement_deterministe() { } ``` -## Fournir des valeurs arbitraires non assertées +## Données d’occurrence arbitraires -Figer épingle une valeur qui compte pour le test. Le besoin symétrique est tout aussi fréquent : une entrée que le test doit fournir mais ne vérifie jamais — un code d’erreur, un message, un instant de survenue. Écrite en dur, elle se lit comme si elle comptait, et une constante réutilisée dans toute une suite peut faire passer un test pour une mauvaise raison. - -`Any` fournit à la place une valeur valide mais arbitraire : la seule entrée qui compte ressort, et les autres s’annoncent comme accessoires : - -```csharp -// Seul le code identifie le scénario ; les messages sont accessoires. -DomainError error = DomainError - .Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) - .WithPublicMessage(Any.ShortMessage()); - -Outcome.Failure(error).ShouldFail().WithCode("ORDER_NOT_FOUND"); -``` - -Ce que `Any` propose : - -| Helper | Renvoie | -| --- | --- | -| `Any.ErrorCode()` | un code non vide, de la forme `ANY_CODE_7F3A9C` | -| `Any.DiagnosticMessage()` / `Any.ShortMessage()` / `Any.DetailedMessage()` | un message non vide | -| `Any.Guid()` / `Any.Instant()` / `Any.String()` / `Any.Int()` / `Any.Bool()` | une primitive arbitraire (`Instant` est en UTC) | -| `Any.Enum()` | un membre quelconque de l’enum (éventuellement un sentinelle comme `Unknown`) | -| `Any.Transience()` / `Any.InteractionDirection()` | une valeur *significative* — jamais le sentinelle `Unknown` | -| `Any.ErrorOrigin()` | un `ErrorOrigin` quelconque | - -Les valeurs sont reconnaissables comme arbitraires (les codes ressemblent à `ANY_CODE_…`), si bien qu’une valeur accessoire apparaissant dans un message d’échec se repère facilement. - -### Reproduire une exécution avec une graine - -Par défaut, les valeurs changent à chaque exécution — ce qui révèle justement un test qui en dépend secrètement. Pour reproduire une exécution précise, épinglez la graine sur une portée ; chaque appel à `Any` à l’intérieur devient déterministe : - -```csharp -using (Any.UseSeed(1234)) { - ErrorCode code = Any.ErrorCode(); // la même valeur à chaque exécution -} -``` - -`Any.UseSeed(...)` suit les mêmes règles de portée que les overrides ci-dessus : jetable, local au contexte, et imbriquable. - -### `OccurredAt` et `InstanceId` arbitraires - -Lorsqu’un test a besoin de données d’occurrence stables sans asserter leurs valeurs exactes, les seams de l’horloge et des identifiants proposent un `UseAny` en pendant de leur `UseFixed` : - -```csharp -using (Clock.UseAny()) { // OccurredAt est figé sur un instant arbitraire - using (InstanceIds.UseAny()) { // chaque erreur reçoit son propre identifiant arbitraire - DomainError error = MakeError(); - // ... assertez le code ou le contexte, pas l’instant ni l’identifiant - } -} -``` - -Les deux acceptent une graine optionnelle (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) pour rendre les valeurs choisies reproductibles. +Lorsqu’un test a besoin que `OccurredAt` et `InstanceId` soient stables sans asserter leurs valeurs exactes, figez-les sur une valeur *arbitraire* avec `Clock.UseAny()` et `InstanceIds.UseAny()` plutôt que `UseFixed` ; les deux acceptent une graine optionnelle pour la reproductibilité. Ils appartiennent à la factory `Any`, plus large, dédiée aux valeurs qu’un test n’assertit pas — voir [Valeurs de test arbitraires](ArbitraryTestValues.fr.md). ## Portée et tests parallèles @@ -254,7 +203,7 @@ Avant d’approuver un test déterministe, vérifiez que : --- --- \ No newline at end of file diff --git a/doc/OperationalIntegration.en.md b/doc/OperationalIntegration.en.md index fc5472b..d1cd799 100644 --- a/doc/OperationalIntegration.en.md +++ b/doc/OperationalIntegration.en.md @@ -237,7 +237,7 @@ Before approving a catalog-delivery pipeline, verify that: --- --- \ No newline at end of file diff --git a/doc/OperationalIntegration.fr.md b/doc/OperationalIntegration.fr.md index 2e3e1a3..4e00a13 100644 --- a/doc/OperationalIntegration.fr.md +++ b/doc/OperationalIntegration.fr.md @@ -237,7 +237,7 @@ Avant d’approuver un pipeline de livraison du catalogue, vérifiez que : --- --- \ No newline at end of file diff --git a/doc/README.fr.md b/doc/README.fr.md index 1903a79..44ad3ab 100644 --- a/doc/README.fr.md +++ b/doc/README.fr.md @@ -163,6 +163,8 @@ Pour les vulnérabilités de sécurité, suivez le processus privé décrit dans - [Cas d’usage](UsagePatterns.fr.md) - [Bonnes pratiques](BestPractices.fr.md) - [Guide des tests](Testing.fr.md) + - [Tests d’erreur déterministes](DeterministicTesting.fr.md) + - [Valeurs de test arbitraires](ArbitraryTestValues.fr.md) ### Générer et exploiter le catalogue diff --git a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md index 07c0d8b..eb5e790 100644 --- a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md +++ b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md @@ -147,6 +147,6 @@ ship test-only surface in the main package. ## References -* `doc/DeterministicTesting.en.md` — the guide where the new surface is documented. +* `doc/ArbitraryTestValues.en.md` — the guide where the new surface is documented. * ADR-0005 — prior naming decision in the same spirit (a name should announce what a call does); context only, not a precedent for this choice. From 8f57b3f237f7ac49608143b4bfa9a8aaaf55584d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 00:01:03 +0000 Subject: [PATCH 4/4] feat(testing): make failing Any runs replayable via Reproducibly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the seed scopes with a single reproducibility runner. `Any.Reproducibly` wraps a test body, pins a fresh seed for the run, and — if the body throws — reports that seed (to Console.Error by default, or an injectable sink such as xUnit's ITestOutputHelper.WriteLine) before rethrowing the original exception. A `Reproducibly(seed, ...)` overload replays a reported seed; a `Func` overload covers async test bodies. Because the runner seeds the ambient Any source, the clock and instance-id seams need no seeded overloads of their own: run `Clock.UseAny()` / `InstanceIds.UseAny()` inside `Reproducibly` and they inherit the seed. So this removes from the public surface: - `Any.UseSeed(int)` (now an internal detail the runner uses); - `Clock.UseAny(int)` and `InstanceIds.UseAny(int)`. The unknown-vs-known-seed distinction drove the change: a known seed is only ever wanted to reproduce a failure, and that is exactly what `Reproducibly` expresses, so a standalone public seed scope had no separate purpose. Tests, the Arbitrary Test Values guide (English and French), the NuGet readme, and ADR-0006 are updated to match; the ADR's "surface the seed for replay" risk is now addressed by the runner. Refs: #135 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012ULkMmGS2EgnALAJtwpxtP --- FirstClassErrors.Testing/Any.cs | 107 +++++++++++++++--- FirstClassErrors.Testing/ArbitrarySource.cs | 10 +- FirstClassErrors.Testing/Clock.cs | 14 +-- FirstClassErrors.Testing/InstanceIds.cs | 18 +-- FirstClassErrors.Testing/README.nuget.md | 3 +- FirstClassErrors.UnitTests/AnyTests.cs | 99 +++++++++------- .../ClockUseAnyTests.cs | 20 ++-- .../InstanceIdsUseAnyTests.cs | 35 +++--- doc/ArbitraryTestValues.en.md | 29 +++-- doc/ArbitraryTestValues.fr.md | 29 +++-- ...rary-test-values-from-a-seedable-source.md | 26 +++-- 11 files changed, 249 insertions(+), 141 deletions(-) diff --git a/FirstClassErrors.Testing/Any.cs b/FirstClassErrors.Testing/Any.cs index 50ea1fd..72c8e92 100644 --- a/FirstClassErrors.Testing/Any.cs +++ b/FirstClassErrors.Testing/Any.cs @@ -9,9 +9,9 @@ namespace FirstClassErrors.Testing; /// /// /// Every value is drawn from a pseudo-random source. By default that source is unseeded, so each run produces -/// fresh values; wrap the code in to make the sequence deterministic and reproducible. -/// The source flows with the current execution context, so it never leaks across tests running in parallel — -/// the same contract the rest of this package keeps. +/// fresh values — which surfaces a test that secretly depends on one. Wrap a value-sensitive test in +/// Reproducibly to make a failing run replayable. The source flows with the current execution context, +/// so it never leaks across tests running in parallel — the same contract the rest of this package keeps. /// /// /// The values are valid (an is never blank, an @@ -29,10 +29,10 @@ namespace FirstClassErrors.Testing; /// DomainError.Create(ErrorCode.Create("ORDER_NOT_FOUND"), Any.DiagnosticMessage()) /// .WithPublicMessage(Any.ShortMessage())); /// -/// // Reproduce a run by pinning the seed: -/// using (Any.UseSeed(1234)) { -/// ErrorCode code = Any.ErrorCode(); // same value on every run -/// } +/// // Make a value-sensitive test replayable: the seed is reported on failure... +/// Any.Reproducibly(() => { /* arrange with Any, act, assert */ }); +/// // ...and replayed by passing it back: +/// Any.Reproducibly(1234, () => { /* ... */ }); /// /// /// @@ -46,18 +46,91 @@ public static class Any { #endregion /// - /// Pins the arbitrary-value source to a deterministic sequence seeded with until the - /// returned scope is disposed, so a test (or a whole run) using becomes reproducible. + /// Runs with the arbitrary-value source pinned to a fresh seed and, if the body throws, + /// reports that seed before letting the exception propagate. This is how a test that draws on + /// stays reproducible: the values still vary between runs (which surfaces accidental dependencies), yet a failure + /// names the exact seed to replay. /// /// - /// Scopes nest: an inner uses its own source and restores the outer one when disposed, - /// without disturbing the outer sequence. Always scope the override with using. Outside any scope the - /// source is unseeded and every run differs. + /// + /// On failure the seed is written to (by default ), + /// with a message naming the Any.Reproducibly(seed, ...) call that reproduces the run. Pass your test + /// framework's output writer (for example xUnit's ITestOutputHelper.WriteLine) to route it there + /// instead. The original exception is rethrown unchanged, so the test still fails with its real message. + /// + /// + /// Reproducing a run needs the same sequence of calls, so a body whose call order depends + /// on non-deterministic external state is not fully replayable from the seed alone. + /// /// - /// The seed that makes the sequence of arbitrary values reproducible. - /// A scope that restores the previous source when disposed. - public static IDisposable UseSeed(int seed) { - return ArbitrarySource.UseSeed(seed); + /// The test body to run under a reproducible arbitrary-value source. + /// The sink the seed is written to on failure. Defaults to when null. + /// Thrown when is null. + public static void Reproducibly(Action body, Action? report = null) { + Reproducibly(ArbitrarySource.NewSeed(), body, report); + } + + /// + /// Replays with the arbitrary-value source pinned to , so a run + /// first seen through the parameterless overload can be + /// reproduced exactly. If the body throws, the seed is reported before the exception propagates. + /// + /// The seed to replay — typically the one a previous failure reported. + /// The test body to run under the seeded arbitrary-value source. + /// The sink the seed is written to on failure. Defaults to when null. + /// Thrown when is null. + public static void Reproducibly(int seed, Action body, Action? report = null) { + if (body is null) { throw new ArgumentNullException(nameof(body)); } + + using (ArbitrarySource.UseSeed(seed)) { + try { + body(); + } catch { + Report(report, seed); + + throw; + } + } + } + + /// + /// Asynchronous counterpart of : awaits + /// under a fresh seed and reports it if the body faults. + /// + /// The asynchronous test body to run under a reproducible arbitrary-value source. + /// The sink the seed is written to on failure. Defaults to when null. + /// A task that completes when completes. + /// Thrown when is null. + public static Task Reproducibly(Func body, Action? report = null) { + return Reproducibly(ArbitrarySource.NewSeed(), body, report); + } + + /// + /// Asynchronous counterpart of : awaits + /// under and reports it if the body faults. + /// + /// The seed to replay — typically the one a previous failure reported. + /// The asynchronous test body to run under the seeded arbitrary-value source. + /// The sink the seed is written to on failure. Defaults to when null. + /// A task that completes when completes. + /// Thrown when is null. + public static async Task Reproducibly(int seed, Func body, Action? report = null) { + if (body is null) { throw new ArgumentNullException(nameof(body)); } + + using (ArbitrarySource.UseSeed(seed)) { + try { + await body().ConfigureAwait(false); + } catch { + Report(report, seed); + + throw; + } + } + } + + private static void Report(Action? report, int seed) { + (report ?? Console.Error.WriteLine)( + $"[FirstClassErrors.Testing] These arbitrary values were seeded with {seed}. Reproduce this run with Any.Reproducibly({seed}, ...)."); } /// @@ -78,7 +151,7 @@ public static bool Bool() { /// /// Returns an arbitrary . Unlike , it is drawn from - /// the seedable source, so it is reproducible inside a scope. + /// the seedable source, so it is reproducible inside an Any.Reproducibly(...) run. /// /// An arbitrary identifier. public static Guid Guid() { diff --git a/FirstClassErrors.Testing/ArbitrarySource.cs b/FirstClassErrors.Testing/ArbitrarySource.cs index cad4622..9c11c3c 100644 --- a/FirstClassErrors.Testing/ArbitrarySource.cs +++ b/FirstClassErrors.Testing/ArbitrarySource.cs @@ -25,7 +25,7 @@ internal static Random Current { get { Random? random = Seeded.Value; if (random is null) { - random = new Random(System.Guid.NewGuid().GetHashCode()); + random = new Random(NewSeed()); Seeded.Value = random; } @@ -33,6 +33,10 @@ internal static Random Current { } } + internal static int NewSeed() { + return System.Guid.NewGuid().GetHashCode(); + } + internal static IDisposable UseSeed(int seed) { Random? previous = Seeded.Value; Seeded.Value = new Random(seed); @@ -83,14 +87,14 @@ internal static TEnum EnumExcluding(params TEnum[] excluded) return pool[Current.Next(pool.Count)]; } - internal static Guid NewGuid(Random random) { + private static Guid NewGuid(Random random) { byte[] bytes = new byte[16]; random.NextBytes(bytes); return new Guid(bytes); } - internal static DateTimeOffset NewInstant(Random random) { + private static DateTimeOffset NewInstant(Random random) { return Origin.AddSeconds(random.Next()); } diff --git a/FirstClassErrors.Testing/Clock.cs b/FirstClassErrors.Testing/Clock.cs index a8aa3b4..408abb7 100644 --- a/FirstClassErrors.Testing/Clock.cs +++ b/FirstClassErrors.Testing/Clock.cs @@ -56,22 +56,12 @@ public static IDisposable UseFixed(DateTimeOffset instant) { /// /// The instant is drawn once, when this method is called, and stays fixed for every error created within the /// scope — the same freezing behavior as . The value comes from 's - /// unseeded source; wrap the call in Any.UseSeed(...), or use the overload, to - /// make it reproducible. + /// source; run the test inside Any.Reproducibly(...) to make the chosen instant reproducible and reported + /// on failure. /// /// A scope that restores the real system clock when disposed. public static IDisposable UseAny() { return UseFixed(ArbitrarySource.Instant()); } - /// - /// Freezes the clock to an arbitrary but reproducible instant, derived from , for - /// the duration of the returned scope. - /// - /// The seed that makes the chosen instant reproducible across runs. - /// A scope that restores the real system clock when disposed. - public static IDisposable UseAny(int seed) { - return UseFixed(ArbitrarySource.NewInstant(new Random(seed))); - } - } diff --git a/FirstClassErrors.Testing/InstanceIds.cs b/FirstClassErrors.Testing/InstanceIds.cs index 691fd18..9fda738 100644 --- a/FirstClassErrors.Testing/InstanceIds.cs +++ b/FirstClassErrors.Testing/InstanceIds.cs @@ -61,25 +61,13 @@ public static IDisposable UseSequential() { /// /// /// Each error created within the scope gets its own fresh arbitrary id (as in production, several errors do not - /// collide), but the ids are drawn from 's source rather than . - /// Wrap the call in Any.UseSeed(...), or use the overload, to make the sequence - /// reproducible. To pin a single fixed id instead, use . + /// collide), drawn from 's source rather than . Run the test + /// inside Any.Reproducibly(...) to make the sequence reproducible and reported on failure. To pin a single + /// fixed id instead, use . /// /// A scope that restores the default (random) identifier when disposed. public static IDisposable UseAny() { return Use(() => ArbitrarySource.Guid()); } - /// - /// Assigns arbitrary but reproducible identifiers, drawn from a sequence seeded with - /// , to the errors created within the scope. - /// - /// The seed that makes the sequence of assigned identifiers reproducible across runs. - /// A scope that restores the default (random) identifier when disposed. - public static IDisposable UseAny(int seed) { - Random random = new(seed); - - return Use(() => ArbitrarySource.NewGuid(random)); - } - } diff --git a/FirstClassErrors.Testing/README.nuget.md b/FirstClassErrors.Testing/README.nuget.md index e3c41b8..384ca27 100644 --- a/FirstClassErrors.Testing/README.nuget.md +++ b/FirstClassErrors.Testing/README.nuget.md @@ -14,7 +14,8 @@ Testing helpers for **FirstClassErrors** — so your tests about errors and outc - **Freezable instance ids** (`InstanceIds.UseFixed(...)`) so `InstanceId` is stable for snapshot and equality assertions. - **Arbitrary values** (`Any.ErrorCode()`, `Any.DiagnosticMessage()`, ...) for the - inputs a test needs but never asserts on — seedable via `Any.UseSeed(...)`, with + inputs a test needs but never asserts on — wrap a value-sensitive test in + `Any.Reproducibly(...)` and a failing run reports the seed to replay, with `Clock.UseAny()` / `InstanceIds.UseAny()` variants of the seams above. Overrides are scoped (`using`), context-local (safe under parallel tests), and never diff --git a/FirstClassErrors.UnitTests/AnyTests.cs b/FirstClassErrors.UnitTests/AnyTests.cs index 69b625c..969c391 100644 --- a/FirstClassErrors.UnitTests/AnyTests.cs +++ b/FirstClassErrors.UnitTests/AnyTests.cs @@ -33,51 +33,76 @@ private static string Batch() { #endregion - [Fact(DisplayName = "UseSeed makes the whole sequence of Any values reproducible.")] - public void UseSeedIsReproducible() { - string first; - string second; + [Fact(DisplayName = "Reproducibly with a given seed replays the same sequence of values.")] + public void ReproduciblyWithASeedIsDeterministic() { + string first = string.Empty; + string second = string.Empty; - using (Any.UseSeed(1234)) { first = Batch(); } - using (Any.UseSeed(1234)) { second = Batch(); } + Any.Reproducibly(1234, () => { first = Batch(); }); + Any.Reproducibly(1234, () => { second = Batch(); }); Check.That(second).IsEqualTo(first); } [Fact(DisplayName = "Different seeds produce different sequences.")] public void DifferentSeedsDiffer() { - string fromOne; - string fromTwo; + string fromOne = string.Empty; + string fromTwo = string.Empty; - using (Any.UseSeed(1)) { fromOne = Batch(); } - using (Any.UseSeed(2)) { fromTwo = Batch(); } + Any.Reproducibly(1, () => { fromOne = Batch(); }); + Any.Reproducibly(2, () => { fromTwo = Batch(); }); Check.That(fromTwo).IsNotEqualTo(fromOne); } - [Fact(DisplayName = "A nested UseSeed scope leaves the outer sequence undisturbed.")] - public void NestedScopeDoesNotDisturbTheOuterSequence() { - int outerFirst; - int outerSecond; - using (Any.UseSeed(7)) { - outerFirst = Any.Int(); - using (Any.UseSeed(99)) { - Any.Int(); - Any.Int(); - } - - outerSecond = Any.Int(); - } + [Fact(DisplayName = "Reproducibly reports the seed and rethrows the original exception on failure.")] + public void ReproduciblyReportsTheSeedAndRethrows() { + string? reported = null; + InvalidOperationException boom = new("boom"); + Action failing = () => throw boom; - int expectedFirst; - int expectedSecond; - using (Any.UseSeed(7)) { - expectedFirst = Any.Int(); - expectedSecond = Any.Int(); - } + InvalidOperationException thrown = Assert.Throws( + () => Any.Reproducibly(4242, failing, message => reported = message)); + + Check.That(ReferenceEquals(thrown, boom)).IsTrue(); + Check.That(reported).IsNotNull(); + Check.That(reported!).Contains("4242"); + } + + [Fact(DisplayName = "Reproducibly does not report when the body succeeds.")] + public void ReproduciblyIsSilentOnSuccess() { + bool reported = false; + + Any.Reproducibly(() => { Any.ErrorCode(); }, _ => reported = true); + + Check.That(reported).IsFalse(); + } + + [Fact(DisplayName = "Reproducibly without a seed reports a replayable seed on failure.")] + public void ReproduciblyWithoutSeedStillReportsAReplayableSeed() { + string? reported = null; + Action failing = () => throw new InvalidOperationException("x"); + + Assert.Throws( + () => Any.Reproducibly(failing, message => reported = message)); + + Check.That(reported).IsNotNull(); + Check.That(reported!).Contains("Any.Reproducibly("); + } + + [Fact(DisplayName = "The async Reproducibly reports the seed and rethrows on failure.")] + public async Task AsyncReproduciblyReportsTheSeedAndRethrows() { + string? reported = null; + + await Assert.ThrowsAsync( + () => Any.Reproducibly(7, async () => { + await Task.Yield(); + + throw new InvalidOperationException("boom"); + }, message => reported = message)); - Check.That(outerFirst).IsEqualTo(expectedFirst); - Check.That(outerSecond).IsEqualTo(expectedSecond); + Check.That(reported).IsNotNull(); + Check.That(reported!).Contains("7"); } [Fact(DisplayName = "ErrorCode returns a non-blank code shaped like ANY_CODE_*.")] @@ -95,19 +120,15 @@ public void InstantIsUtc() { [Fact(DisplayName = "Transience never returns the Unknown sentinel.")] public void TransienceExcludesUnknown() { - using (Any.UseSeed(0)) { - for (int i = 0; i < 200; i++) { - Check.That(Any.Transience()).IsNotEqualTo(Transience.Unknown); - } + for (int i = 0; i < 200; i++) { + Check.That(Any.Transience()).IsNotEqualTo(Transience.Unknown); } } [Fact(DisplayName = "InteractionDirection never returns the Unknown sentinel.")] public void InteractionDirectionExcludesUnknown() { - using (Any.UseSeed(0)) { - for (int i = 0; i < 200; i++) { - Check.That(Any.InteractionDirection()).IsNotEqualTo(InteractionDirection.Unknown); - } + for (int i = 0; i < 200; i++) { + Check.That(Any.InteractionDirection()).IsNotEqualTo(InteractionDirection.Unknown); } } diff --git a/FirstClassErrors.UnitTests/ClockUseAnyTests.cs b/FirstClassErrors.UnitTests/ClockUseAnyTests.cs index 26b78ec..8305ade 100644 --- a/FirstClassErrors.UnitTests/ClockUseAnyTests.cs +++ b/FirstClassErrors.UnitTests/ClockUseAnyTests.cs @@ -31,20 +31,24 @@ public void UseAnyFreezesASingleInstant() { } } - [Fact(DisplayName = "Clock.UseAny(seed) picks the same instant for a given seed.")] - public void UseAnyWithSeedIsReproducible() { - DateTimeOffset first; - DateTimeOffset second; - - using (Clock.UseAny(42)) { first = AnError().OccurredAt; } - using (Clock.UseAny(42)) { second = AnError().OccurredAt; } + [Fact(DisplayName = "Inside Any.Reproducibly, Clock.UseAny picks the same instant for a given seed.")] + public void UseAnyIsReproducibleUnderAReproduciblyScope() { + DateTimeOffset first = default; + DateTimeOffset second = default; + + Any.Reproducibly(42, () => { + using (Clock.UseAny()) { first = AnError().OccurredAt; } + }); + Any.Reproducibly(42, () => { + using (Clock.UseAny()) { second = AnError().OccurredAt; } + }); Check.That(second).IsEqualTo(first); } [Fact(DisplayName = "Clock.UseAny only affects code inside the scope; the real clock resumes after disposal.")] public void UseAnyIsScoped() { - using (Clock.UseAny(1)) { AnError(); } + using (Clock.UseAny()) { AnError(); } DateTimeOffset before = DateTimeOffset.UtcNow; DateTimeOffset live = AnError().OccurredAt; diff --git a/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs b/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs index ca7d864..ee8097c 100644 --- a/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs +++ b/FirstClassErrors.UnitTests/InstanceIdsUseAnyTests.cs @@ -31,22 +31,25 @@ public void UseAnyAssignsDistinctIds() { } } - [Fact(DisplayName = "InstanceIds.UseAny(seed) reproduces the same sequence of ids.")] - public void UseAnyWithSeedIsReproducible() { - Guid firstRunA; - Guid firstRunB; - Guid secondRunA; - Guid secondRunB; - - using (InstanceIds.UseAny(7)) { - firstRunA = AnError().InstanceId; - firstRunB = AnError().InstanceId; - } - - using (InstanceIds.UseAny(7)) { - secondRunA = AnError().InstanceId; - secondRunB = AnError().InstanceId; - } + [Fact(DisplayName = "Inside Any.Reproducibly, InstanceIds.UseAny reproduces the same id sequence for a given seed.")] + public void UseAnyIsReproducibleUnderAReproduciblyScope() { + Guid firstRunA = Guid.Empty; + Guid firstRunB = Guid.Empty; + Guid secondRunA = Guid.Empty; + Guid secondRunB = Guid.Empty; + + Any.Reproducibly(7, () => { + using (InstanceIds.UseAny()) { + firstRunA = AnError().InstanceId; + firstRunB = AnError().InstanceId; + } + }); + Any.Reproducibly(7, () => { + using (InstanceIds.UseAny()) { + secondRunA = AnError().InstanceId; + secondRunB = AnError().InstanceId; + } + }); Check.That(secondRunA).IsEqualTo(firstRunA); Check.That(secondRunB).IsEqualTo(firstRunB); diff --git a/doc/ArbitraryTestValues.en.md b/doc/ArbitraryTestValues.en.md index bc85438..bd1e201 100644 --- a/doc/ArbitraryTestValues.en.md +++ b/doc/ArbitraryTestValues.en.md @@ -52,20 +52,29 @@ The guarantees stop at type validity. A helper does not target a domain precondi Use `Any.Enum()` when any member will do — a sentinel included — and the named enum helpers when the test needs a value that actually drives the behavior under test. -## Reproduce a run with a seed +## Reproduce a failing run The source is unseeded by default, so the values differ between runs. That is deliberate: a test that passes only for one particular value is relying on something it never states, and varying the value surfaces that coupling. -The cost is reproducibility. The default seed is **not surfaced today**, so a run that failed on a particular unseeded value cannot be replayed from its output alone. Keep the arbitrary value out of what decides pass or fail — or, when a specific run must be reproducible, pin a seed on the narrowest useful scope; a suite-wide seed is fine when full stability is preferable to variation between runs. Every `Any` call inside the scope then becomes deterministic: +When a run matters enough to reproduce, wrap the test body in `Any.Reproducibly`. It pins a fresh seed for the run and, if the body throws, **reports that seed** before the failure propagates — so a red test tells you exactly how to replay it: ```csharp -using (Any.UseSeed(1234)) { - ErrorCode first = Any.ErrorCode(); - ErrorCode second = Any.ErrorCode(); // the same two values on every run -} +[Fact] +public void Some_value_sensitive_test() => + Any.Reproducibly(() => { + // ... arrange with Any, act, assert ... + }); +``` + +On failure the seed is written to `Console.Error` by default; pass your framework's writer (for example xUnit's `ITestOutputHelper.WriteLine`) to route it there instead. Replay the run by handing the reported seed back: + +```csharp +Any.Reproducibly(1234, () => { + // ... the same body ... +}); ``` -Seeds nest: an inner `Any.UseSeed(...)` uses its own sequence and restores the outer one when it exits. Outside any scope the source is unseeded and every run differs. +Reproducing a run needs the same sequence of `Any` calls, so a body whose call order depends on non-deterministic external state is not fully replayable from the seed alone. There is also an asynchronous overload, `Any.Reproducibly(Func)`, for `async` test bodies. ## Arbitrary `OccurredAt` and `InstanceId` @@ -85,11 +94,11 @@ using (InstanceIds.UseAny()) { } ``` -Both take an optional seed (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) to make the chosen values reproducible. To pin a *specific* instant or id instead, use `UseFixed` — see [Deterministic Error Tests](DeterministicTesting.en.md). +Both draw from the same source as `Any`, so running them inside `Any.Reproducibly` makes their instant and ids reproducible too. To pin a *specific* instant or id instead, use `UseFixed` — see [Deterministic Error Tests](DeterministicTesting.en.md). ## Scope and parallel tests -`Any.UseSeed`, `Clock.UseAny`, and `InstanceIds.UseAny` all take effect only inside their `using` block and are restored when the scope exits. The seeded source is stored in an `AsyncLocal`, so it follows the test's own execution flow and never leaks into other tests running at the same time. +`Any.Reproducibly`, `Clock.UseAny`, and `InstanceIds.UseAny` all take effect only for the run or `using` block they wrap, and the arbitrary source is restored when it exits. That source is stored in an `AsyncLocal`, so it follows the test's own execution flow and never leaks into other tests running at the same time. ## Review checklist @@ -98,7 +107,7 @@ Before reaching for an arbitrary value, verify that: - the value does **not** change the functional path the test exercises — it must not feed a branch, a validation, a serialization, or an ordering, even indirectly; - the value is genuinely not checked by the test — otherwise use a literal; - a named enum helper is used when the test needs a meaningful value, rather than `Any.Enum()`; -- a seed is pinned whenever a failing run would otherwise be irreproducible; +- a value-sensitive test is wrapped in `Any.Reproducibly` so a failing run reports the seed to replay; - `Clock.UseAny` / `InstanceIds.UseAny` are used for stable-but-irrelevant occurrence data, and `UseFixed` when the exact value is asserted. --- diff --git a/doc/ArbitraryTestValues.fr.md b/doc/ArbitraryTestValues.fr.md index d2a4d91..dbeec29 100644 --- a/doc/ArbitraryTestValues.fr.md +++ b/doc/ArbitraryTestValues.fr.md @@ -52,20 +52,29 @@ Les garanties s’arrêtent à la validité du type. Un helper ne vise aucune pr Utilisez `Any.Enum()` quand n’importe quel membre convient — sentinelle comprise — et les helpers d’enum nommés quand le test a besoin d’une valeur qui déclenche réellement le comportement concerné. -## Reproduire une exécution avec une graine +## Reproduire une exécution en échec La source n’est pas seedée par défaut : les valeurs diffèrent donc d’une exécution à l’autre. C’est délibéré : un test qui ne passe que pour une valeur particulière dépend de quelque chose qu’il n’énonce pas, et faire varier la valeur révèle ce couplage. -Le coût, c’est la reproductibilité. La graine par défaut **n’est pas exposée aujourd’hui** : une exécution ayant échoué sur une valeur non seedée précise ne peut donc pas être rejouée à partir de sa seule sortie. Gardez la valeur arbitraire hors de ce qui décide du succès ou de l’échec — ou, lorsqu’une exécution précise doit être reproductible, épinglez une graine sur la portée la plus étroite utile ; une graine commune à toute la suite convient lorsque la stabilité complète est préférable à la variation entre les exécutions. Chaque appel à `Any` dans la portée devient alors déterministe : +Quand une exécution mérite d’être reproduite, enveloppez le corps du test dans `Any.Reproducibly`. La méthode épingle une graine fraîche pour l’exécution et, si le corps lève une exception, **rapporte cette graine** avant de laisser l’échec se propager — un test rouge te dit ainsi exactement comment le rejouer : ```csharp -using (Any.UseSeed(1234)) { - ErrorCode first = Any.ErrorCode(); - ErrorCode second = Any.ErrorCode(); // les deux mêmes valeurs à chaque exécution -} +[Fact] +public void Some_value_sensitive_test() => + Any.Reproducibly(() => { + // ... arrange avec Any, act, assert ... + }); +``` + +En cas d’échec, la graine est écrite sur `Console.Error` par défaut ; passe le writer de ton framework (par exemple l’`ITestOutputHelper.WriteLine` de xUnit) pour l’y router. Rejoue l’exécution en redonnant la graine rapportée : + +```csharp +Any.Reproducibly(1234, () => { + // ... le même corps ... +}); ``` -Les graines s’imbriquent : un `Any.UseSeed(...)` interne utilise sa propre séquence et restaure celle de l’extérieur à sa sortie. Hors de toute portée, la source n’est pas seedée et chaque exécution diffère. +Reproduire une exécution nécessite la **même séquence** d’appels à `Any` : un corps dont l’ordre des tirages dépend d’un état externe non déterministe n’est pas entièrement rejouable à partir de la seule graine. Une surcharge asynchrone, `Any.Reproducibly(Func)`, existe pour les corps de test `async`. ## `OccurredAt` et `InstanceId` arbitraires @@ -85,11 +94,11 @@ using (InstanceIds.UseAny()) { } ``` -Les deux acceptent une graine optionnelle (`Clock.UseAny(1234)`, `InstanceIds.UseAny(1234)`) pour rendre les valeurs choisies reproductibles. Pour épingler un instant ou un identifiant *précis*, utilisez `UseFixed` — voir [Tests d’erreur déterministes](DeterministicTesting.fr.md). +Les deux tirent de la même source qu’`Any` : les exécuter à l’intérieur d’un `Any.Reproducibly` rend donc leur instant et leurs identifiants reproductibles eux aussi. Pour épingler un instant ou un identifiant *précis*, utilisez `UseFixed` — voir [Tests d’erreur déterministes](DeterministicTesting.fr.md). ## Portée et tests parallèles -`Any.UseSeed`, `Clock.UseAny` et `InstanceIds.UseAny` ne prennent effet qu’à l’intérieur de leur bloc `using` et sont restaurés à la sortie de la portée. La source seedée est stockée dans un `AsyncLocal` : elle suit le flux d’exécution du test lui-même et ne fuit jamais dans d’autres tests s’exécutant en même temps. +`Any.Reproducibly`, `Clock.UseAny` et `InstanceIds.UseAny` ne prennent effet que pour l’exécution ou le bloc `using` qu’ils enveloppent, et la source arbitraire est restaurée à leur sortie. Cette source est stockée dans un `AsyncLocal` : elle suit le flux d’exécution du test lui-même et ne fuit jamais dans d’autres tests s’exécutant en même temps. ## Checklist de revue @@ -98,7 +107,7 @@ Avant de recourir à une valeur arbitraire, vérifiez que : - la valeur ne **modifie pas** le chemin fonctionnel exercé par le test — elle ne doit alimenter ni une branche, ni une validation, ni une sérialisation, ni un classement, même indirectement ; - la valeur n’est réellement pas vérifiée par le test — sinon utilisez un littéral ; - un helper d’enum nommé est utilisé quand le test a besoin d’une valeur significative, plutôt que `Any.Enum()` ; -- une graine est épinglée dès qu’une exécution en échec serait sinon irreproductible ; +- un test sensible aux valeurs est enveloppé dans `Any.Reproducibly`, pour qu’une exécution en échec rapporte la graine à rejouer ; - `Clock.UseAny` / `InstanceIds.UseAny` servent pour des données d’occurrence stables mais sans importance, et `UseFixed` lorsque la valeur exacte est assertée. --- diff --git a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md index eb5e790..52e578b 100644 --- a/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md +++ b/maintainers/adr/0006-supply-arbitrary-test-values-from-a-seedable-source.md @@ -44,8 +44,9 @@ one, a failing run cannot be replayed. `FirstClassErrors.Testing` supplies arbitrary test values through a single, dependency-free, context-local pseudo-random source whose determinism is opt-in -via a seed scope, and the clock and instance-id seams gain `UseAny` variants -layered on that same source. +through a `Reproducibly` runner that seeds a run, reports the seed on failure, and +replays it on demand; the clock and instance-id seams gain unseeded `UseAny` +variants that draw from that same source. ## Rationale @@ -59,10 +60,12 @@ layered on that same source. seams already keep. The new surface is the same shape as the surface it sits next to, rather than a second, unrelated mechanism. * **Arbitrary by default, reproducible on demand.** An unseeded default makes - values differ between runs, which is what exposes overfitting; an opt-in seed - scope makes a chosen test or run replayable. This directly serves the - legibility and overfitting facts, and the reproducibility requirement, without - forcing every test to manage a seed. + values differ between runs, which is what exposes overfitting; wrapping a + value-sensitive test in `Reproducibly` pins a seed, reports it when the body + fails, and replays it when given one — so a failing run is recoverable without + forcing every test to manage a seed. Because the seed lives on the run that + owns execution, the standalone clock and instance-id `UseAny` scopes need no + seeded overload of their own: run inside `Reproducibly`, they inherit the seed. * **The name carries the intent.** Reaching for an explicitly *arbitrary* value reads as "this input is incidental", which is the distinction a hand-picked literal cannot make. The `UseAny` variants extend the existing `UseFixed` / @@ -128,9 +131,10 @@ ship test-only surface in the main package. coincidentally start from the same seed, their "arbitrary" values coincide. This is harmless (the values are not asserted on) and is mitigated by deriving the default seed from a fresh identifier per context. -* Reproducing a failure still depends on the author having set a seed; an - unseeded failing test is not replayable from its output alone. Surfacing the - auto-chosen seed for replay is a possible later refinement. +* Reproducing a failure requires the author to have wrapped the test in + `Reproducibly`, and holds only while the body's sequence of `Any` calls is + deterministic; an ordinary unwrapped test that fails on an arbitrary value is + still not replayable from its output alone. * Until enforced by tooling, the "arbitrary value ⇒ use the source, not a literal" habit holds only through review and documentation. @@ -139,7 +143,9 @@ ship test-only surface in the main package. * Document the surface in the testing guide, in English and French, in lockstep. * Consider a design for arbitrary `ErrorContextKey` values that respects the process-wide registry. -* Consider surfacing the auto-chosen seed so an unseeded failure can be replayed. +* Consider an optional test-framework adapter (for example an xUnit + `[ReproducibleFact]`) so the seed is surfaced automatically, without wrapping + each body in `Reproducibly`. * Consider an instance-based generator if callers ask for an explicit object. * Consider extracting the generic value engine into a standalone, error-agnostic utility if a second consumer appears; it is kept internally separable from the