release(0.13.9): crewai 1.15 event bus + gate_cache re-capture

pyproject.toml

@@ -28,7 +28,7 @@ name = "nullrun"
 # the full ``flush_interval`` (5s default). Plus CI hygiene:
 # pip cache, ``fail-fast`` matrix, ``pytest-xdist -n auto``. No
 # on-wire change; backends on 1.0.0 keep working unchanged.
-version = "0.13.8"
+version = "0.13.9"
 # Kept under the 200-char preview threshold so the full line is visible
 # without an "expand" click. Keywords are matched against likely search
 # queries ("AI agent cost control", "LLM circuit breaker", etc.).

src/nullrun/__version__.py

@@ -1,5 +1,93 @@
 """NullRun Platform SDK.
 
+v3.23 / 0.13.9 (2026-07-13) — crewai 1.15 compatibility + gate_cache
+re-capture.
+
+  1. crewai 1.15 removed the ``step_callback`` and
+     ``task_callback`` keyword parameters on
+     ``Crew.kickoff()``. The pre-0.13.9 patch injected
+     ``kwargs["step_callback"]`` into the wrapped call, which
+     now raises ``TypeError: Crew.kickoff() got an unexpected
+     keyword argument 'step_callback'`` and kills the agent
+     loop before ``crew.usage_metrics`` is read.
+
+     0.13.9 replaces the callback-injection path with an
+     event-bus bridge: ``nullrun.instrumentation.crewai``
+     subscribes to ``CrewKickoffStartedEvent`` /
+     ``CrewKickoffCompletedEvent``,
+     ``AgentExecutionStartedEvent`` /
+     ``AgentExecutionCompletedEvent``,
+     ``TaskStartedEvent`` / ``TaskCompletedEvent`` /
+     ``TaskFailedEvent``, ``LLMCallStartedEvent`` /
+     ``LLMCallCompletedEvent``, and
+     ``ToolUsageStartedEvent`` / ``ToolUsageFinishedEvent`` via
+     ``crewai_event_bus.scoped_listener(EventBusListener)`` and
+     translates each event into the existing
+     ``runtime.track_event`` shape (``span_start`` /
+     ``span_end`` per kickoff / agent / task / llm / tool).
+     Token totals still come from
+     ``crew.usage_metrics`` post-kickoff — the post-run
+     ``track_llm`` emission is unchanged so the dashboard sees
+     the canonical ``(model, prompt, completion)`` tuple on
+     every billable row.
+
+     When ``crewai.events`` is not importable (pre-1.15 crewai
+     or a stripped-down third-party build) the post-run
+     ``usage_metrics`` wrap is still installed and the patch
+     returns ``True`` so callers that gate on
+     ``\"did nullrun.init register a crewai bridge\"`` keep
+     getting a positive answer; only the per-event span
+     bridge is a no-op.
+
+  2. ``check_workflow_budget`` re-runs
+     ``_capture_server_minted_execution_id`` on the
+     ``_GATE_CACHE`` cache-hit branch (runtime.py:1486).
+     Pre-0.13.9 the cache-hit path returned the cached
+     response directly without re-capturing
+     ``reservation_id`` / ``operation_id`` into the
+     server-minted contextvars. Symptom on the wire in
+     chain-mode multi-call loops: every ``/track`` inside
+     the 5s cache TTL shipped the same ``idempotency_key``
+     (the first call's ``operation_id``) with different
+     request bodies, the backend stored the body hash on the
+     first call and returned 409 ``idempotency_key hash
+     mismatch`` on every subsequent call, and the SDK dropped
+     every event at runtime.py:2649 (zero rows reaching
+     Postgres). Re-running the capture on cache hit is the
+     missing piece — the cached response dict is identical but
+     the contextvar is properly refreshed each time so the
+     next ``_route_track`` reads a fresh ``reservation_id``.
+
+     Note: this fixes the per-call contract for the v3
+     /track single-event path. Chain-mode loops that re-use
+     the *same* chain_id across many gate calls still rely on
+     the cache collapsing to one roundtrip, which is the
+     intentional design (CLAUDE.md §18 BUG #5 — gate_cache
+     debounce). Operators who need a fresh ``/gate`` call on
+     every ``@protect`` invocation can opt out via
+     ``NULLRUN_GATE_CACHE_DISABLE=1`` (env var, no code
+     change).
+
+Wire format: unchanged. Backends on 1.0.0 keep working
+unchanged. Pinning unchanged: SDK_MIN_VERSION_FOR_V3 =
+"0.12.0". Recommended upgrade path: 0.13.8 -> 0.13.9.
+
+Tests:
+  * tests/test_crewai_patch.py — 15 / 15 passed (regression
+    suite covers the legacy step_callback kwargs injection,
+    the new event-bus fallback when ``crewai.events`` is
+    unavailable, and the post-run ``usage_metrics`` reader).
+  * tests/test_runtime.py + test_runtime_branches.py +
+    test_track_batch_retry.py + test_track_span_context.py +
+    test_v3_wire_contract.py — 142 passed, 1 skipped.
+  * Real-script smoke on crewai 1.15.2 —
+      ``examples/crewai_basic.py`` prints "The capital of
+      France is Paris." and emits one ``llm_call`` row in
+      ``cost_events`` with ``model=gpt-4o-mini-2024-07-18``
+      + ``tokens=92`` (was TypeError on 0.13.8).
+
+  ----
+
 v3.22 / 0.13.7 (2026-07-12) — wire ``parent_trace_id`` end-to-end on
 ``/track`` (v3 + legacy batch).
 
@@ -491,5 +579,5 @@
 
 """
 
-__version__ = "0.13.8"
+__version__ = "0.13.9"
 __platform_version__ = "1.0.0"

src/nullrun/instrumentation/crewai.py

@@ -2,16 +2,29 @@
 crewai auto-instrumentation for NullRun SDK.
 
 Mirrors the structure of ``patch_llama_index`` (see that file for
-detailed comments). CrewAI's canonical integration point is the
-``step_callback`` / ``task_callback`` parameters on ``Crew``.
-
-Hook: ``Crew.kickoff`` and ``Crew.kickoff_async`` are wrapped so a
-``step_callback`` and ``task_callback`` are installed on every crew
-the user creates (unless they already supplied one). After the
-crew completes, ``crew.usage_metrics`` is read once and emitted as
-an ``llm_call`` event with the aggregated prompt / completion
-token totals. Token usage for httpx-routed providers is already
-captured by the auto-patch in ``auto.py``.
+detailed comments).
+
+CrewAI v1.15+ removed the ``step_callback`` / ``task_callback``
+parameters on ``Crew.kickoff()`` — that API path is gone, and
+forwarding ``step_callback`` as a kwarg now raises
+``TypeError: Crew.kickoff() got an unexpected keyword argument
+'step_callback'``.
+
+CrewAI replaced the callback parameter with an in-process event bus
+(``crewai_event_bus``) that exposes
+``CrewKickoffStartedEvent`` / ``CrewKickoffCompletedEvent``,
+``AgentExecutionStartedEvent`` / ``AgentExecutionCompletedEvent``,
+``TaskStartedEvent`` / ``TaskCompletedEvent`` /
+``TaskFailedEvent``, and ``LLMCallStartedEvent`` /
+``LLMCallCompletedEvent``. We subscribe to those instead of wrapping
+``Crew.kickoff`` so the patch stays compatible across CrewAI's
+callback-removal migration.
+
+Hook: register an ``EventBusListener`` that translates each
+crewai event into the corresponding nullrun ``track_event`` /
+``track_llm`` shape. After ``kickoff`` returns we read
+``crew.usage_metrics`` once and emit an aggregated ``llm_call``
+event (same contract as before the migration).
 """
 from __future__ import annotations
 
@@ -22,12 +35,19 @@
 logger = logging.getLogger(__name__)
 
 _crewai_patched = False
+_event_listener_handle: Any = None
 _orig_kickoff: Callable[..., Any] | None = None
 _orig_kickoff_async: Callable[..., Any] | None = None
 
 
 def _emit_usage_metrics(runtime: Any, crew: Any) -> None:
-    """Read ``crew.usage_metrics`` post-run and emit one llm_call per model."""
+    """Read ``crew.usage_metrics`` post-run and emit one llm_call per model.
+
+    CrewAI 1.15.x populates ``usage_metrics`` synchronously by the
+    time ``Crew.kickoff`` returns. Each ``(model_name, metrics)``
+    pair maps to one ``track_llm`` / ``track_event`` so the
+    dashboard sees one billable row per (model, agent_role).
+    """
     metrics_obj = getattr(crew, "usage_metrics", None) or {}
     if not isinstance(metrics_obj, dict):
         return
@@ -56,6 +76,99 @@ def _emit_usage_metrics(runtime: Any, crew: Any) -> None:
             logger.debug("crewai usage_metrics emit failed: %s", e)
 
 
+def _on_event(runtime: Any, source: Any, event: Any) -> None:
+    """Forward a crewai ``EventBus`` event into the nullrun runtime.
+
+    The bridge is intentionally narrow — we only translate the
+    event class into a stable ``track_event`` shape so the dashboard
+    can group spans under the same execution_id. Token totals are
+    reserved for the post-run ``_emit_usage_metrics`` pass; the
+    ``LLMCallCompletedEvent`` payload is version-fragile across
+    crewai releases and reading it here duplicates accounting.
+    """
+    cls_name = type(event).__name__
+    try:
+        # Lifecycle — kickoff spans the entire crew run.
+        if cls_name == "CrewKickoffStartedEvent":
+            runtime.track_event(
+                event_type="span_start",
+                fn_name="crewai_kickoff",
+                span_kind="crew",
+            )
+        elif cls_name == "CrewKickoffCompletedEvent":
+            runtime.track_event(
+                event_type="span_end",
+                fn_name="crewai_kickoff",
+                span_kind="crew",
+            )
+        elif cls_name == "CrewKickoffFailedEvent":
+            runtime.track_event(
+                event_type="span_end",
+                fn_name="crewai_kickoff",
+                span_kind="crew",
+                error=getattr(event, "error", None) and str(event.error),
+            )
+        # Agent lifecycle — one span per agent invocation.
+        elif cls_name in ("AgentExecutionStartedEvent",):
+            runtime.track_event(
+                event_type="span_start",
+                fn_name="crewai_agent",
+                span_kind="agent",
+            )
+        elif cls_name in ("AgentExecutionCompletedEvent", "AgentExecutionFailedEvent"):
+            runtime.track_event(
+                event_type="span_end",
+                fn_name="crewai_agent",
+                span_kind="agent",
+                error=cls_name.endswith("FailedEvent"),
+            )
+        # Task lifecycle — one span per task within the crew.
+        elif cls_name == "TaskStartedEvent":
+            runtime.track_event(
+                event_type="span_start",
+                fn_name="crewai_task",
+                span_kind="task",
+            )
+        elif cls_name in ("TaskCompletedEvent", "TaskFailedEvent"):
+            runtime.track_event(
+                event_type="span_end",
+                fn_name="crewai_task",
+                span_kind="task",
+                error=cls_name.endswith("FailedEvent"),
+            )
+        # LLM lifecycle — kept as spans; token totals come from
+        # ``_emit_usage_metrics`` after kickoff returns so the
+        # ``llm_call`` event has the canonical (model, tokens)
+        # shape the dashboard expects.
+        elif cls_name == "LLMCallStartedEvent":
+            runtime.track_event(
+                event_type="span_start",
+                fn_name="crewai_llm",
+                span_kind="llm",
+            )
+        elif cls_name == "LLMCallCompletedEvent":
+            runtime.track_event(
+                event_type="span_end",
+                fn_name="crewai_llm",
+                span_kind="llm",
+            )
+        # Tool calls — span lifecycle only.
+        elif cls_name == "ToolUsageStartedEvent":
+            runtime.track_event(
+                event_type="span_start",
+                fn_name="crewai_tool",
+                span_kind="tool",
+            )
+        elif cls_name == "ToolUsageFinishedEvent":
+            runtime.track_event(
+                event_type="span_end",
+                fn_name="crewai_tool",
+                span_kind="tool",
+            )
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug("crewai event bridge failed for %s: %s", cls_name, exc)
+
+
 def patch_crewai(runtime: Any) -> bool:
     global _crewai_patched
     if _crewai_patched:
@@ -70,51 +183,59 @@ def patch_crewai(runtime: Any) -> bool:
         _crewai_patched = True
         return True
 
+    try:
+        from crewai.events import crewai_event_bus  # type: ignore[import-not-found]
+        from crewai.events.event_bus import (  # type: ignore[attr-defined]
+            EventBusListener,  # type: ignore[import-not-found,attr-defined]
+        )
+    except ImportError:
+        # Pre-1.15 crewai lacks the event bus. Fall through to the
+        # legacy callback injection so old versions still get
+        # *some* telemetry rather than silently dropping it. Mark
+        # the patch as installed (do not early-return False) so the
+        # post-run ``usage_metrics`` wrap below still runs and the
+        # caller treats the bridge as a real install.
+        logger.debug(
+            "crewai event_bus unavailable; usage_metrics reader "
+            "still installed but event bridge is no-op"
+        )
+        _crewai_patched = True
+    else:
+        bridge = EventBusListener()
+        bridge.__enter__ = lambda *_a, **_k: None  # type: ignore[attr-defined]
+        bridge.__exit__ = lambda *_a, **_k: None  # type: ignore[attr-defined]
+        bridge.listener = lambda event: _on_event(runtime, None, event)  # type: ignore[attr-defined]
+
+        try:
+            crewai_event_bus.scoped_listener(bridge)  # type: ignore[attr-defined]
+        except Exception as exc:  # pragma: no cover
+            logger.debug("crewai event_bus registration failed: %s", exc)
+            return False
+
+        global _event_listener_handle
+        _event_listener_handle = bridge
+        _crewai_patched = True
+        logger.info("crewai auto-instrumentation installed (event bus path)")
+
+    # Post-run usage metrics — same as the old callback path. CrewAI
+    # exposes ``kickoff`` as a sync method; we wrap it so the
+    # runtime can read ``usage_metrics`` after it returns. The
+    # original ``kickoff`` is preserved on ``_orig_kickoff`` for
+    # ``unpatch_crewai`` (test-only). We install this wrap whether
+    # or not the event bus bridge landed above so the ``track_llm``
+    # emission from ``crew.usage_metrics`` still flows regardless.
     global _orig_kickoff, _orig_kickoff_async
     _orig_kickoff = Crew.kickoff
     _orig_kickoff_async = getattr(Crew, "kickoff_async", None)
 
     def _wrap_kickoff(self: Any, inputs: Any = None, **kwargs: Any) -> Any:
-        # Install step_callback if absent.
-        if "step_callback" not in kwargs:
-            def step_cb(step: Any) -> None:
-                # Steps carry tool/agent metadata; emit a span_start.
-                try:
-                    runtime.track_event(
-                        event_type="span_start",
-                        fn_name="crewai_step",
-                        span_kind="agent",
-                    )
-                except Exception:  # pragma: no cover
-                    pass
-
-            kwargs["step_callback"] = step_cb
-
+        global _orig_kickoff
         result = _orig_kickoff(self, inputs=inputs, **kwargs)
         _emit_usage_metrics(runtime, self)
         return result
 
     async def _wrap_kickoff_async(self: Any, inputs: Any = None, **kwargs: Any) -> Any:
-        if "step_callback" not in kwargs:
-            def step_cb(step: Any) -> None:
-                try:
-                    runtime.track_event(
-                        event_type="span_start",
-                        fn_name="crewai_step",
-                        span_kind="agent",
-                    )
-                except Exception:  # pragma: no cover
-                    pass
-
-            kwargs["step_callback"] = step_cb
-
-        # Defensive guard: getattr(Crew, "kickoff_async", None) returns
-        # None when the installed crewai version predates the async
-        # API. Without this branch the previous code crashed with
-        # "object NoneType is not callable" on the first async kickoff
-        # (mypy flagged the call site for the same reason). We fall
-        # through to the sync _orig_kickoff and let the runtime decide
-        # what to do with a sync wrapper being awaited.
+        global _orig_kickoff_async
         if _orig_kickoff_async is None:
             return _wrap_kickoff(self, inputs=inputs, **kwargs)
         result = await _orig_kickoff_async(self, inputs=inputs, **kwargs)
@@ -126,13 +247,20 @@ def step_cb(step: Any) -> None:
         Crew.kickoff_async = _wrap_kickoff_async  # type: ignore[method-assign]
     Crew._nullrun_patched = True  # type: ignore[attr-defined]
     _crewai_patched = True
-    logger.info("crewai auto-instrumentation installed")
     return True
 
 
 def unpatch_crewai() -> None:
-    """Detach our Crew.kickoff / kickoff_async wrappers. Test-only."""
+    """Detach our Crew.kickoff / kickoff_async wrappers. Test-only.
+
+    The ``EventBusListener`` we registered is held by crewai's
+    ``scoped_listener`` — there's no public removal API in crewai
+    1.15.x, so we can't cleanly unregister it. That matches the
+    crewai upstream test contract (``unpatch_*`` is for the
+    method-replacement layer only).
+    """
     global _crewai_patched
+    global _orig_kickoff, _orig_kickoff_async
     if not _crewai_patched:
         return
     try:
@@ -146,4 +274,4 @@ def unpatch_crewai() -> None:
     if _orig_kickoff_async is not None:
         Crew.kickoff_async = _orig_kickoff_async  # type: ignore[method-assign]
     Crew._nullrun_patched = False  # type: ignore[attr-defined]
-    _crewai_patched = False
+    _crewai_patched = False