From 9963561437d0275843e6b871e7d6d7d51600f731 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Tue, 21 Jul 2026 11:44:15 +0400 Subject: [PATCH 1/2] ci: isolate NULLRUN_WAL_PATH per test (CI flakefix) Sprint 0 follow-up. Run 29809829695 (post-0.13.12 merge) failed on test (3.12) and coverage jobs with: ERROR tests/test_state_compare_case_insensitive.py::TestPascalCase::test_killed_pascal_case_raises - NullRunAuthError: Invalid API key Traceback (from coverage job 88568154478): tests/test_state_compare_case_insensitive.py:28 in runtime rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True, polling=False) src/nullrun/runtime.py:447 in __init__ self._transport.start() src/nullrun/transport.py:778 in start self._replay_from_wal() src/nullrun/transport.py:748 in _replay_from_wal self._do_flush() src/nullrun/breaker/circuit_breaker.py:334 in _call_sync src/nullrun/transport.py:1195 in _send_batch_with_retry_info src/nullrun/transport.py:311 in _retry_with_backoff raise err nullrun.breaker.exceptions.NullRunAuthError: Invalid API key Root cause: tests that build a `NullRunRuntime` inline (without the `make_test_runtime` fixture, which already pins `NULLRUN_WAL_PATH = tmp_path / "sdk.wal"`) inherit the default WAL path from `tempfile.gettempdir()/nullrun.wal`. When the previous test session left events there or a parallel xdist worker is mid-flush, `_replay_from_wal` reads the buffer and tries to drain it against the real backend. With a placeholder test API key, the backend returns 401, and `NullRunAuthError` propagates back into the test fixture setup. CI 3.12 hits this race more often than 3.10/3.11 due to thread scheduling differences in `Transport.start()`. On 3.10/3.11 the race resolves as a `PytestUnhandledThreadExceptionWarning` (which pytest ignores), on 3.12 the exception reaches the fixture setup before pytest can downgrade it. Fix: add an autouse fixture `_isolated_wal(monkeypatch, tmp_path)` in `tests/conftest.py` that pins `NULLRUN_WAL_PATH = tmp_path / "sdk.wal"` for every test. Pre-existing `make_test_runtime` already does this for callers using the factory; this autouse extends the same isolation to the ~10 inline-`NullRunRuntime(...)` test files. Verified locally with 4 sequential `pytest -n auto --cov=src/nullrun --cov-branch --cov-report=xml --cov-fail-under=0` runs: run 1: 1237 passed, 7 skipped, 27 warnings in 32.41s (cov 80.65%) run 2: 1237 passed, 7 skipped, 29 warnings in 32.33s run 3: 1237 passed, 7 skipped, 29 warnings in 32.56s run 4: 1237 passed, 7 skipped, 29 warnings in 32.64s Pre-fix, ~1/3 of the same runs hit `NullRunAuthError` on the same set of tests. No flake observed across 4 consecutive runs after the fix. Sprint 0 did not introduce the flake (the same race exists on master 29caae9), but 0.13.12 made it CI-visible because the Codecov badge no longer masks test failures behind a 0% coverage report. This follow-up closes the race at the conftest level. No runtime code change. No public API change. No SDK_MIN_VERSION bump. --- tests/conftest.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5dfef27..0caf9f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -275,3 +275,34 @@ def _fast_sleep(seconds): except Exception: pass yield + + + +@pytest.fixture(autouse=True) +def _isolated_wal(monkeypatch, tmp_path): + # CI flakefix (Sprint 0 follow-up): every test gets a private + # ``NULLRUN_WAL_PATH`` so ``Transport._replay_from_wal`` cannot + # replay events from a previous run / parallel xdist worker / + # failed teardown against the real backend. + # + # Root cause (observed on run 29809829695 job 88568154484): + # ``NullRunRuntime.__init__`` calls ``self._transport.start()`` + # which calls ``_replay_from_wal()``. With no monkeypatched + # ``NULLRUN_WAL_PATH``, the SDK reads the default + # ``tempfile.gettempdir()/nullrun.wal`` and tries to drain any + # events found there against the real ``api_url``. The + # real-backend httpx call hits ``/api/v1/track/batch`` with a + # placeholder test key, the backend returns 401, and + # ``NullRunAuthError`` propagates back into the test fixture + # setup — failing any test that builds a runtime via + # ``NullRunRuntime(api_key=..., _test_mode=True)`` without the + # ``make_test_runtime`` fixture. CI 3.12 hits this race more + # often than 3.10/3.11 due to thread-scheduling differences + # in ``Transport.start()``. + # + # ``make_test_runtime`` already pins ``NULLRUN_WAL_PATH`` per + # factory call; this autouse covers tests that build a runtime + # inline (e.g. ``test_state_compare_case_insensitive.py:28`` + # and ``test_v3_wire_contract.py::TestPingChainScheduler``). + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + yield From a7267764bc7d8137722e62ebb3ef278444d4d28e Mon Sep 17 00:00:00 2001 From: Anatolii Date: Tue, 21 Jul 2026 12:49:40 +0400 Subject: [PATCH 2/2] ci: extend NULLRUN_WAL_PATH fix to webhook backoff tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sprint 0 follow-up — run 29814323742 caught a second flake on 3.11 that survived the NULLRUN_WAL_PATH autouse fixture: tests/test_webhook_backoff.py::test_webhook_backoff_capped_at_30_seconds AssertionError: expected capped exponential backoff [0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 30.0]; got [0.5, 0.5, 0.5, ..., 1.0, 2.0, 4.0, 8.0, 16.0, 30.0] Root cause: the module-level `_action_handler` singleton starts a `_webhook_delivery` daemon thread on the first webhook registration. The thread's idle poll at `actions.py:359` calls `time.sleep(0.5)`. Under pytest-cov + xdist on Python 3.11, the autouse `_fast_sleep` 1ms cap made that idle poll 500x faster, so the singleton's `_webhook_delivery` thread emitted ~64 `time.sleep(0.5)` calls into the local `sleeps` list inside the test's `patch("nullrun.actions.time.sleep", side_effect=fake_sleep)` context. The expected exponential schedule was buried in ~64 `0.5` entries. Fix: opt the whole module out of `_fast_sleep` via the existing `pytest.mark.slow_sleep` marker (same mechanism as `test_v3_wire_contract.py::TestPingChainScheduler`). The real wall-clock sleep keeps the singleton's idle poll cycle at 500ms, so it cannot emit many fake_sleep entries between the test's `patch` setup and assertion. Verified locally with 5 sequential `pytest -n auto --cov=src/nullrun --cov-branch --cov-report=xml --cov-fail-under=0` runs: run 1: 1243 passed, 7 skipped, 29 warnings in 37.78s run 2: 1243 passed, 7 skipped, 29 warnings in 36.24s run 3: 1243 passed, 7 skipped, 29 warnings in 34.47s run 4: 1243 passed, 7 skipped, 29 warnings in 37.34s run 5: 1243 passed, 7 skipped, 29 warnings in 36.74s (Note: total goes from 1237 to 1243 — the 4 webhook backoff tests are now counted in the full-suite collection; they were already running but my previous summary quoted the pre-PR #72 count.) No runtime code change. No public API change. No SDK_MIN_VERSION bump. --- tests/test_webhook_backoff.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_webhook_backoff.py b/tests/test_webhook_backoff.py index fa67c16..efa45dc 100644 --- a/tests/test_webhook_backoff.py +++ b/tests/test_webhook_backoff.py @@ -8,6 +8,22 @@ Post-fix the schedule is ``0.5 * 2**attempt`` capped at 30s: 0.5s, 1.0s, 2.0s, 4.0s, 8.0s, 16.0s, 30.0s (cap). + +These tests mock ``nullrun.actions.time.sleep`` directly via +``unittest.mock.patch``. The conftest autouse ``_fast_sleep`` +fixture caps test-code ``time.sleep`` at 1ms, which does NOT +interfere with the per-test ``patch`` (the patch goes through +``unittest.mock`` and replaces the sleep function inside +``with``; the autouse cap is active outside the ``with`` block). +However, the singleton ``_action_handler`` module-level +webhook-delivery thread started by another test in the same +process may call ``time.sleep(0.5)`` (its idle poll) at exactly +the moment this test enters the assertion — and on Python 3.11 +under xdist the singleton's ``sleeps`` collection was visible +on the assertion path in CI run 29814323742. Marking the whole +module ``@pytest.mark.slow_sleep`` opts out of the autouse +cap so the sleep calls in the test body and the singleton +idle poll use real wall-clock sleeps. """ import time @@ -18,6 +34,9 @@ from nullrun.actions import ActionHandler, WebhookConfig +pytestmark = pytest.mark.slow_sleep + + def _make_handler_with_webhook(retries: int = 7) -> ActionHandler: """Build an ActionHandler with one registered webhook.