From 9963561437d0275843e6b871e7d6d7d51600f731 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Tue, 21 Jul 2026 11:44:15 +0400 Subject: [PATCH] ci: isolate NULLRUN_WAL_PATH per test (CI flakefix) Sprint 0 follow-up. Run 29809829695 (post-0.13.12 merge) failed on test (3.12) and coverage jobs with: ERROR tests/test_state_compare_case_insensitive.py::TestPascalCase::test_killed_pascal_case_raises - NullRunAuthError: Invalid API key Traceback (from coverage job 88568154478): tests/test_state_compare_case_insensitive.py:28 in runtime rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True, polling=False) src/nullrun/runtime.py:447 in __init__ self._transport.start() src/nullrun/transport.py:778 in start self._replay_from_wal() src/nullrun/transport.py:748 in _replay_from_wal self._do_flush() src/nullrun/breaker/circuit_breaker.py:334 in _call_sync src/nullrun/transport.py:1195 in _send_batch_with_retry_info src/nullrun/transport.py:311 in _retry_with_backoff raise err nullrun.breaker.exceptions.NullRunAuthError: Invalid API key Root cause: tests that build a `NullRunRuntime` inline (without the `make_test_runtime` fixture, which already pins `NULLRUN_WAL_PATH = tmp_path / "sdk.wal"`) inherit the default WAL path from `tempfile.gettempdir()/nullrun.wal`. When the previous test session left events there or a parallel xdist worker is mid-flush, `_replay_from_wal` reads the buffer and tries to drain it against the real backend. With a placeholder test API key, the backend returns 401, and `NullRunAuthError` propagates back into the test fixture setup. CI 3.12 hits this race more often than 3.10/3.11 due to thread scheduling differences in `Transport.start()`. On 3.10/3.11 the race resolves as a `PytestUnhandledThreadExceptionWarning` (which pytest ignores), on 3.12 the exception reaches the fixture setup before pytest can downgrade it. Fix: add an autouse fixture `_isolated_wal(monkeypatch, tmp_path)` in `tests/conftest.py` that pins `NULLRUN_WAL_PATH = tmp_path / "sdk.wal"` for every test. Pre-existing `make_test_runtime` already does this for callers using the factory; this autouse extends the same isolation to the ~10 inline-`NullRunRuntime(...)` test files. Verified locally with 4 sequential `pytest -n auto --cov=src/nullrun --cov-branch --cov-report=xml --cov-fail-under=0` runs: run 1: 1237 passed, 7 skipped, 27 warnings in 32.41s (cov 80.65%) run 2: 1237 passed, 7 skipped, 29 warnings in 32.33s run 3: 1237 passed, 7 skipped, 29 warnings in 32.56s run 4: 1237 passed, 7 skipped, 29 warnings in 32.64s Pre-fix, ~1/3 of the same runs hit `NullRunAuthError` on the same set of tests. No flake observed across 4 consecutive runs after the fix. Sprint 0 did not introduce the flake (the same race exists on master 29caae9), but 0.13.12 made it CI-visible because the Codecov badge no longer masks test failures behind a 0% coverage report. This follow-up closes the race at the conftest level. No runtime code change. No public API change. No SDK_MIN_VERSION bump. --- tests/conftest.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5dfef27..0caf9f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -275,3 +275,34 @@ def _fast_sleep(seconds): except Exception: pass yield + + + +@pytest.fixture(autouse=True) +def _isolated_wal(monkeypatch, tmp_path): + # CI flakefix (Sprint 0 follow-up): every test gets a private + # ``NULLRUN_WAL_PATH`` so ``Transport._replay_from_wal`` cannot + # replay events from a previous run / parallel xdist worker / + # failed teardown against the real backend. + # + # Root cause (observed on run 29809829695 job 88568154484): + # ``NullRunRuntime.__init__`` calls ``self._transport.start()`` + # which calls ``_replay_from_wal()``. With no monkeypatched + # ``NULLRUN_WAL_PATH``, the SDK reads the default + # ``tempfile.gettempdir()/nullrun.wal`` and tries to drain any + # events found there against the real ``api_url``. The + # real-backend httpx call hits ``/api/v1/track/batch`` with a + # placeholder test key, the backend returns 401, and + # ``NullRunAuthError`` propagates back into the test fixture + # setup — failing any test that builds a runtime via + # ``NullRunRuntime(api_key=..., _test_mode=True)`` without the + # ``make_test_runtime`` fixture. CI 3.12 hits this race more + # often than 3.10/3.11 due to thread-scheduling differences + # in ``Transport.start()``. + # + # ``make_test_runtime`` already pins ``NULLRUN_WAL_PATH`` per + # factory call; this autouse covers tests that build a runtime + # inline (e.g. ``test_state_compare_case_insensitive.py:28`` + # and ``test_v3_wire_contract.py::TestPingChainScheduler``). + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + yield