diff --git a/sdk/agentserver/azure-ai-agentserver-core/CHANGELOG.md b/sdk/agentserver/azure-ai-agentserver-core/CHANGELOG.md index 9b594533e897..335a81f85327 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/CHANGELOG.md +++ b/sdk/agentserver/azure-ai-agentserver-core/CHANGELOG.md @@ -8,6 +8,17 @@ ### Breaking Changes +- The resilient task subsystem is now **strictly opt-in**. `AgentServerHost` + constructs the `TaskManager` only when resilient tasks are enabled via + `set_resilient_tasks_enabled(True)` (or a protocol option that maps to it, + e.g. the responses `resilient_background`). Previously the manager was always + constructed and a declared `@task` / `@multi_turn_task` implicitly enabled the + startup recovery scan. Now, declaring a task does **not** turn the subsystem + on: with the switch off, `get_task_manager()` raises `TaskManagerNotInitialized` + and `.run()` / `.start()` cannot run a task. Existing apps that rely on `@task` + must call `set_resilient_tasks_enabled(True)` (before host startup) to keep + durable tasks and crash recovery. Plain servers that use no tasks are + unaffected and continue to pay nothing. - Removed `TaskMetadata`, `TaskContext.metadata`, and `TaskRun.metadata`. Durable application state now belongs in an explicit `FoundryStateStore` and no longer shares task lifecycle PATCHes or lease renewal. Typed task diff --git a/sdk/agentserver/azure-ai-agentserver-core/README.md b/sdk/agentserver/azure-ai-agentserver-core/README.md index 5bb689abac11..12b170a2042b 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/README.md +++ b/sdk/agentserver/azure-ai-agentserver-core/README.md @@ -191,8 +191,17 @@ python my_agent.py The `@task` decorator builds crash-resilient agents that survive container restarts, OOM kills, and redeployments. Task state is persisted to a task store, enabling automatic recovery and multi-turn suspend/resume patterns. +The resilient task subsystem is **opt-in**: call `set_resilient_tasks_enabled(True)` (before host startup, typically at import time) so the framework constructs the `TaskManager` and runs crash recovery. Without it, `.run()` / `.start()` raise `TaskManagerNotInitialized`. + ```python -from azure.ai.agentserver.core.tasks import task, TaskContext +from azure.ai.agentserver.core.tasks import ( + set_resilient_tasks_enabled, + task, + TaskContext, +) + +# Opt in to the durable task subsystem (required for @task to run). +set_resilient_tasks_enabled(True) @task(name="process_document") async def process_document(ctx: TaskContext[dict]) -> dict: diff --git a/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/_base.py b/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/_base.py index b2f578865a8e..19eba248380b 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/_base.py +++ b/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/_base.py @@ -311,38 +311,30 @@ async def _lifespan(_app: Starlette) -> AsyncGenerator[None, None]: # noqa: RUF # --- Resilient task manager initialization --- # - # The TaskManager is CONSTRUCTED unconditionally (whenever the - # resilient tasks module is importable). Construction is cheap and - # makes NO task-store calls — it only builds in-memory state (an - # idle provider client, empty routing tables, a lease-owner string). - # This keeps ``get_task_manager()`` working so a ``@task``-based app - # can run tasks without hitting ``TaskManagerNotInitialized``, - # while paying zero network cost until a task is actually used. + # The resilient ``TaskManager`` is CONSTRUCTED ONLY when resilient + # tasks were explicitly enabled via + # ``set_resilient_tasks_enabled(True)``. Recovery (and the durable + # task subsystem as a whole) is strictly opt-in: a host that does + # not set the switch pays nothing — no manager, no provider client, + # no task-store call, no recovery loop. # - # The network-backed startup RECOVERY SCAN (a blocking hosted - # task-store ``list()`` plus ``DefaultAzureCredential`` token - # acquisition, which would otherwise gate server readiness) — and - # the periodic recovery loop it spawns — run when EITHER holds: - # (1) at least one durable task was declared (``@task`` / - # ``@multi_turn_task``, tracked in ``_REGISTERED_DESCRIPTORS``) - # — an app that uses tasks gets recovery automatically, OR - # (2) resilient tasks were explicitly enabled via - # ``set_resilient_tasks_enabled(True)`` — a force-enable that - # starts the recovery loop even before any task is declared, - # so a task declared later is picked up by the loop. - # A plain server that neither declares a task nor sets the switch - # (e.g. an invocations-only host) makes no task-store call at - # startup. - # - # NOTE (deferred): if the switch is OFF and no task is declared at - # startup, the recovery loop is not started, so a task declared - # LATER in that lifetime will run but its prior-crash orphans are - # not scanned until the next restart. Fully closing that requires a - # lazy manager-start on first late registration; tracked as future - # work (the manager cannot be made fully async, only lazily - # started). + # When the switch is OFF the manager is NOT installed, so + # ``get_task_manager()`` raises ``TaskManagerNotInitialized``. + # Callers that route through the manager (e.g. the responses + # protocol's ``store=true`` path) are expected to SWALLOW that + # signal and degrade to non-durable, in-process execution — the + # response still runs and persists, it simply is not crash- + # recoverable. Declaring a durable task (``@task`` / + # ``@multi_turn_task``) does NOT implicitly turn the subsystem on; + # an app that wants durable tasks / recovery must set the switch. task_manager = None - try: + if _resilient_tasks_enabled(): + # Resilient tasks were explicitly enabled. Construct the manager + # and run startup recovery. If EITHER fails, durability was + # explicitly requested, so fail the lifespan (fail-fast at boot) + # rather than start a server that would silently run store=true + # work non-durably. This guarantees that in a running deployment + # "enabled" always implies a live manager. from .tasks._manager import ( # pylint: disable=import-outside-toplevel TaskManager, set_task_manager, @@ -354,20 +346,31 @@ async def _lifespan(_app: Starlette) -> AsyncGenerator[None, None]: # noqa: RUF shutdown_grace_seconds=_read_task_manager_shutdown_grace(), ) set_task_manager(task_manager) - - if _resilient_tasks_enabled() or _has_registered_tasks(): + try: await task_manager.startup() - logger.info("TaskManager initialized with startup recovery") - else: - logger.info( - "TaskManager initialized (recovery deferred; enabled=%s, tasks_declared=%s)", - _resilient_tasks_enabled(), - _has_registered_tasks(), - ) - except ImportError: - pass # resilient module not available - except Exception: # pylint: disable=broad-exception-caught - logger.warning("Failed to initialize TaskManager", exc_info=True) + except BaseException: + # ``set_task_manager`` above already installed the manager + # globally, but startup failed and the lifespan will now + # abort before the shutdown block runs. Best-effort tear the + # partially-started manager down and clear the singleton so a + # failed/half-initialized manager is not left visible through + # ``get_task_manager()``; then re-raise to fail the lifespan. + try: + await task_manager.shutdown() + except Exception: # pylint: disable=broad-exception-caught + logger.warning("Error shutting down TaskManager after startup failure", exc_info=True) + set_task_manager(None) + task_manager = None + raise + logger.info("TaskManager initialized with startup recovery") + else: + logger.info( + "TaskManager NOT initialized (resilient tasks disabled; enable via " + "set_resilient_tasks_enabled(True)). Durable tasks and crash recovery " + "are inactive; store=true work degrades to non-durable execution. " + "tasks_declared=%s", + _has_registered_tasks(), + ) yield diff --git a/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_enablement.py b/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_enablement.py index 84dae9c48963..c90e84360b75 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_enablement.py +++ b/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_enablement.py @@ -1,24 +1,24 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -"""Explicit force-enable switch for the resilient task recovery scan. +"""Explicit opt-in switch for the resilient task subsystem. -The resilient ``TaskManager`` is always constructed by ``AgentServerHost`` (a -cheap, in-memory object that makes no task-store calls), so ``get_task_manager`` -and ``.run()`` / ``.start()`` work regardless of this switch. What this switch -affects is only the network-backed **startup recovery scan** (a hosted -task-store ``list()`` plus credential-token acquisition) and the periodic -recovery loop it spawns. +The resilient ``TaskManager`` is constructed by ``AgentServerHost`` **only when +this switch is on**. It is the single source of truth for the durable task +subsystem: when the switch is off, no ``TaskManager`` is installed, so +``get_task_manager()`` raises +:class:`~azure.ai.agentserver.core.tasks.TaskManagerNotInitialized` and +``.run()`` / ``.start()`` cannot run a task. When on, the manager is +constructed and the startup crash-recovery scan (plus the periodic recovery +loop) runs. -That recovery runs at startup when EITHER a durable task has been declared -(``@task`` / ``@multi_turn_task``) OR this switch is on. So an app that uses -tasks gets recovery automatically; this switch is a **force-enable** that -starts the recovery loop even before any task is declared (useful when tasks -are registered lazily after startup — the running loop then picks them up). +Recovery — and durable tasks as a whole — is therefore strictly opt-in. Merely +declaring a durable task (``@task`` / ``@multi_turn_task``) does **not** turn +the subsystem on; you must set this switch (directly, or via a protocol option +that maps to it, e.g. the responses ``resilient_background`` server option). -The switch is process-global and defaults to ``False``. It is intentionally -decoupled from any ``AgentServerHost`` instance so it can be flipped -independently at import time, e.g.:: +The switch is process-global and defaults to ``False``. Set it before +``AgentServerHost`` lifespan startup (typically at import time):: from azure.ai.agentserver.core.tasks import set_resilient_tasks_enabled @@ -32,18 +32,22 @@ @experimental def set_resilient_tasks_enabled(value: bool = True) -> None: - """Force-enable (or clear) the resilient task recovery scan process-wide. + """Opt in to (or clear) the resilient task subsystem process-wide. - Setting this to ``True`` starts the startup recovery scan + periodic - recovery loop even when no durable task is declared at startup. It does - NOT gate the ``TaskManager``'s existence: ``.run()`` / ``.start()`` work - whether or not this is set — this only controls automatic crash recovery. + This gates whether ``AgentServerHost`` constructs the ``TaskManager`` at + lifespan startup. Setting it to ``True`` constructs the manager, runs the + startup crash-recovery scan, and starts the periodic recovery loop. When it + is ``False`` no manager is installed: ``get_task_manager()`` raises + :class:`~azure.ai.agentserver.core.tasks.TaskManagerNotInitialized` and a + durable task cannot run (callers such as the responses ``store=true`` path + swallow that and degrade to non-durable in-process execution). Must be called before ``AgentServerHost`` lifespan startup (typically at import time) to take effect. Defaults to enabling when called with no argument. - :param value: ``True`` to force-enable recovery, ``False`` to clear. + :param value: ``True`` to enable the resilient task subsystem, ``False`` to + clear. :type value: bool """ global _RESILIENT_TASKS_ENABLED # pylint: disable=global-statement @@ -52,11 +56,11 @@ def set_resilient_tasks_enabled(value: bool = True) -> None: @experimental def resilient_tasks_enabled() -> bool: - """Return whether the recovery scan was explicitly force-enabled. + """Return whether the resilient task subsystem is enabled. - Note this reflects only the switch — recovery also runs automatically when - a durable task is declared, so a ``False`` return does not mean recovery is - off, nor that tasks are unavailable. + This is the authoritative gate: a ``False`` return means no ``TaskManager`` + is constructed and durable tasks / crash recovery are inactive (declaring a + task does NOT change this — the subsystem is opt-in). :return: ``True`` if :func:`set_resilient_tasks_enabled` turned it on. :rtype: bool diff --git a/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_manager.py b/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_manager.py index 1cff24b64b0a..0600ec43b670 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_manager.py +++ b/sdk/agentserver/azure-ai-agentserver-core/azure/ai/agentserver/core/tasks/_manager.py @@ -297,8 +297,11 @@ def get_task_manager() -> TaskManager: ) raise TaskManagerNotInitialized( - "TaskManager not initialized. Ensure resilient tasks " - "are enabled on the AgentServerHost." # pylint: disable=implicit-str-concat + "TaskManager not initialized: the resilient task subsystem is not enabled. " + "Durable tasks and crash recovery are opt-in - call " + "set_resilient_tasks_enabled(True) before host startup (e.g. at import time), " + "or, for the responses protocol, construct the host with " + "ResponsesServerOptions(resilient_background=True)." # pylint: disable=implicit-str-concat ) return _manager diff --git a/sdk/agentserver/azure-ai-agentserver-core/docs/tasks-guide.md b/sdk/agentserver/azure-ai-agentserver-core/docs/tasks-guide.md index 5de9ab44010f..686482afc2fc 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/docs/tasks-guide.md +++ b/sdk/agentserver/azure-ai-agentserver-core/docs/tasks-guide.md @@ -115,31 +115,31 @@ What this primitive deliberately does **not** do: ### Enabling resilient tasks -The resilient `TaskManager`'s **startup recovery scan** — a network round-trip -to the hosted task store that reclaims tasks left in-flight by a crashed prior -instance — runs at startup when **either** of the following holds: - -1. at least one durable task has been declared (`@task` / `@multi_turn_task`) — - an app that uses tasks gets recovery automatically, **or** -2. it was explicitly force-enabled via `set_resilient_tasks_enabled(True)`. - -The force-enable is useful when tasks are registered *lazily* (declared after -startup): it starts the periodic recovery loop up front so a task declared -later is still recovered. +The resilient task subsystem is **opt-in**. The `TaskManager` is constructed — +and its **startup recovery scan** (a network round-trip to the hosted task store +that reclaims tasks left in-flight by a crashed prior instance) plus the +periodic recovery loop run — **only when the switch is on**: ```python from azure.ai.agentserver.core.tasks import set_resilient_tasks_enabled -set_resilient_tasks_enabled(True) # force-enable recovery before any task +set_resilient_tasks_enabled(True) # opt in BEFORE host startup (e.g. at import) ``` Use `resilient_tasks_enabled()` to read the current switch state. -The `TaskManager` itself is always constructed (a cheap, in-memory object that -makes no task-store calls until a task is used), so `get_task_manager()` and -`.run()` / `.start()` work regardless of the switch. A server that neither -declares a task nor sets the switch (e.g. an invocations-only host) simply -skips the startup recovery scan and pays none of its latency. +Merely declaring a durable task (`@task` / `@multi_turn_task`) does **not** turn +the subsystem on. When the switch is off, no `TaskManager` is installed: +`get_task_manager()` raises `TaskManagerNotInitialized` and `.run()` / `.start()` +cannot run a task. A server that does not set the switch (e.g. an +invocations-only host) constructs no manager and pays none of the recovery-scan +latency. + +> **Protocol note:** the responses protocol exposes a `resilient_background` +> server option that maps to this switch — constructing a +> `ResponsesAgentServerHost(options=ResponsesServerOptions(resilient_background=True))` +> enables the subsystem automatically, so responses apps typically don't call +> `set_resilient_tasks_enabled` directly. ### One-shot diff --git a/sdk/agentserver/azure-ai-agentserver-core/tests/tasks/test_task_manager_optin.py b/sdk/agentserver/azure-ai-agentserver-core/tests/tasks/test_task_manager_optin.py index 6cff55e7d56f..8d0150c78a57 100644 --- a/sdk/agentserver/azure-ai-agentserver-core/tests/tasks/test_task_manager_optin.py +++ b/sdk/agentserver/azure-ai-agentserver-core/tests/tasks/test_task_manager_optin.py @@ -1,21 +1,20 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -"""Tests for the gated resilient ``TaskManager`` startup recovery scan. - -``AgentServerHost`` always constructs the resilient ``TaskManager`` (a cheap, -in-memory object that makes no task-store calls), so ``get_task_manager()`` and -``.run()`` / ``.start()`` work regardless. Its network-backed **startup -recovery scan** (and the periodic recovery loop it spawns) runs when EITHER: - -1. at least one durable task has been declared (``@task`` / - ``@multi_turn_task``, tracked in the ``_REGISTERED_DESCRIPTORS`` list), OR -2. the switch was set via ``set_resilient_tasks_enabled(True)`` (default - ``False``) — a force-enable. - -Both signals are read directly at lifespan startup. When neither is true, no -task-store call is made — plain servers (e.g. invocations-only hosts) pay -nothing. +"""Tests for the gated resilient ``TaskManager`` construction + recovery. + +The resilient ``TaskManager`` is CONSTRUCTED ONLY when resilient tasks were +explicitly enabled via ``set_resilient_tasks_enabled(True)`` (default +``False``). Recovery — and the durable task subsystem as a whole — is strictly +opt-in. + +When the switch is off the manager is NOT installed, so ``get_task_manager()`` +raises ``TaskManagerNotInitialized``; callers that route through the manager +(e.g. the responses ``store=true`` path) swallow that at their outer catch and +degrade to non-durable in-process execution. Merely declaring a durable task +(``@task`` / ``@multi_turn_task``) — including internal protocol primitives — +does NOT turn the subsystem on. A plain server pays nothing: no manager, no +task-store call. """ import logging @@ -27,6 +26,7 @@ ) from azure.ai.agentserver.core.tasks import ( TaskContext, + TaskManagerNotInitialized, multi_turn_task, resilient_tasks_enabled, set_resilient_tasks_enabled, @@ -136,63 +136,64 @@ async def _probe(ctx: "TaskContext[dict]") -> None: class TestLifespanManagerAndRecovery: - """The TaskManager is ALWAYS constructed (cheap, no task-store calls); - the network-backed startup recovery scan runs when EITHER the switch is - enabled OR at least one durable task is declared.""" + """The resilient TaskManager is CONSTRUCTED ONLY when the switch is + explicitly enabled via ``set_resilient_tasks_enabled(True)``. With the + switch off no manager is installed and ``get_task_manager()`` raises — + callers swallow that and degrade to non-durable execution.""" @pytest.mark.asyncio - async def test_neither_enabled_nor_task_no_recovery( + async def test_neither_enabled_nor_task_no_manager( self, _clean_state, _fake_task_manager ) -> None: - """No switch, no task: the manager is constructed and installed so - ``get_task_manager()`` works (no ``TaskManagerNotInitialized``) — but - the startup recovery scan does NOT run (plain invocations host).""" + """No switch, no task: the manager is NOT constructed and + ``get_task_manager()`` raises ``TaskManagerNotInitialized``.""" from azure.ai.agentserver.core import AgentServerHost from azure.ai.agentserver.core.tasks._manager import get_task_manager app = AgentServerHost() async with app.router.lifespan_context(app): - # A manager exists and is retrievable during the active lifespan. - assert len(_fake_task_manager.instances) == 1 - assert get_task_manager() is _fake_task_manager.instances[0] - # No recovery scan happened (neither gate true). - assert _fake_task_manager.instances[0].startup_called is False - - # Torn down + cleared on shutdown. - assert _fake_task_manager.instances[0].shutdown_called is True + # No manager constructed (switch off). + assert len(_fake_task_manager.instances) == 0 + with pytest.raises(TaskManagerNotInitialized): + get_task_manager() @pytest.mark.asyncio - async def test_task_declared_runs_recovery_without_switch(self, _clean_state, _fake_task_manager) -> None: - """A declared task alone runs recovery (backward compatible — an - existing ``@task`` app gets recovery without calling the switch).""" + async def test_task_declared_without_switch_no_manager(self, _clean_state, _fake_task_manager) -> None: + """A declared task alone does NOT construct the manager: the durable + task subsystem is opt-in and gated solely on the switch.""" from azure.ai.agentserver.core import AgentServerHost + from azure.ai.agentserver.core.tasks._manager import get_task_manager _declare_task() # switch left at default False app = AgentServerHost() async with app.router.lifespan_context(app): - pass - assert len(_fake_task_manager.instances) == 1 - assert _fake_task_manager.instances[0].startup_called is True + assert len(_fake_task_manager.instances) == 0 + with pytest.raises(TaskManagerNotInitialized): + get_task_manager() @pytest.mark.asyncio - async def test_switch_alone_runs_recovery_without_task(self, _clean_state, _fake_task_manager) -> None: - """The switch alone runs recovery (force-enable) — starting the - periodic recovery loop so a task declared later is picked up.""" + async def test_switch_alone_builds_manager_and_runs_recovery(self, _clean_state, _fake_task_manager) -> None: + """The switch constructs the manager and runs recovery (force-enable) + — starting the periodic recovery loop so a task declared later is + picked up.""" from azure.ai.agentserver.core import AgentServerHost + from azure.ai.agentserver.core.tasks._manager import get_task_manager set_resilient_tasks_enabled(True) app = AgentServerHost() async with app.router.lifespan_context(app): - pass - assert len(_fake_task_manager.instances) == 1 - assert _fake_task_manager.instances[0].startup_called is True + assert len(_fake_task_manager.instances) == 1 + assert get_task_manager() is _fake_task_manager.instances[0] + assert _fake_task_manager.instances[0].startup_called is True + assert _fake_task_manager.instances[0].shutdown_called is True @pytest.mark.asyncio - async def test_switch_and_task_runs_recovery( + async def test_switch_and_task_builds_manager_and_runs_recovery( self, _clean_state, _fake_task_manager, caplog: pytest.LogCaptureFixture ) -> None: - """Both signals true -> manager built and startup recovery runs.""" + """Switch on and a task declared -> manager built and startup recovery + runs.""" from azure.ai.agentserver.core import AgentServerHost set_resilient_tasks_enabled(True) @@ -210,9 +211,13 @@ async def test_switch_and_task_runs_recovery( assert any("TaskManager initialized with startup recovery" in r.message for r in caplog.records) @pytest.mark.asyncio - async def test_multi_turn_task_runs_recovery_without_switch(self, _clean_state, _fake_task_manager) -> None: - """A declared ``@multi_turn_task`` alone also runs recovery.""" + async def test_multi_turn_task_declared_without_switch_no_manager( + self, _clean_state, _fake_task_manager + ) -> None: + """A declared ``@multi_turn_task`` alone also does NOT construct the + manager (opt-in).""" from azure.ai.agentserver.core import AgentServerHost + from azure.ai.agentserver.core.tasks._manager import get_task_manager @multi_turn_task(name="gate_lifespan_mt") async def _probe(ctx: "TaskContext[dict]") -> None: @@ -220,7 +225,46 @@ async def _probe(ctx: "TaskContext[dict]") -> None: app = AgentServerHost() async with app.router.lifespan_context(app): - pass + assert len(_fake_task_manager.instances) == 0 + with pytest.raises(TaskManagerNotInitialized): + get_task_manager() - assert len(_fake_task_manager.instances) == 1 - assert _fake_task_manager.instances[0].startup_called is True + @pytest.mark.asyncio + async def test_switch_on_startup_failure_fails_lifespan( + self, _clean_state, monkeypatch: pytest.MonkeyPatch + ) -> None: + """When resilient tasks are ENABLED but manager startup fails, the + lifespan must fail fast (fail-fast at boot) rather than start a server + that would silently run store=true work non-durably. The partially + started manager must be torn down and the singleton cleared so it is not + left visible through ``get_task_manager()``.""" + from azure.ai.agentserver.core import AgentServerHost + from azure.ai.agentserver.core.tasks._manager import get_task_manager + + shutdown_calls: list[int] = [] + + class _FailingManager: + def __init__(self, **kwargs) -> None: + pass + + async def startup(self) -> None: + raise RuntimeError("simulated boot failure") + + async def shutdown(self) -> None: + shutdown_calls.append(1) + + monkeypatch.setattr( + "azure.ai.agentserver.core.tasks._manager.TaskManager", + _FailingManager, + ) + set_resilient_tasks_enabled(True) + + app = AgentServerHost() + with pytest.raises(RuntimeError, match="simulated boot failure"): + async with app.router.lifespan_context(app): + pass + + # The partially started manager was torn down and the singleton cleared. + assert shutdown_calls == [1] + with pytest.raises(TaskManagerNotInitialized): + get_task_manager() diff --git a/sdk/agentserver/azure-ai-agentserver-responses/CHANGELOG.md b/sdk/agentserver/azure-ai-agentserver-responses/CHANGELOG.md index 312ac5bb7990..c06d63abf27e 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/CHANGELOG.md +++ b/sdk/agentserver/azure-ai-agentserver-responses/CHANGELOG.md @@ -4,6 +4,16 @@ ### Breaking Changes +- The durable-response subsystem is now **opt-in**. A `store=true` response is + wrapped in a resilient task (with crash recovery) only when the resilient task + subsystem is enabled — which `resilient_background=True` (or + `set_resilient_tasks_enabled(True)`) now does automatically. On a host that + enables neither, `store=true` responses run **non-durably in-process**: they + execute and persist (GET works), but a response in-flight when the process is + ungracefully killed stays `in_progress` on a later GET (no mark-failed/recovery) + — matching a plain stateless server. A one-time startup log announces which + mode is active. Previously every responses host implicitly used the task + subsystem (and paid the boot recovery scan) regardless of these options. - Removed `ResponseContext.conversation_chain_metadata` and the `ConversationChainMetadataNamespace` protocol. Resilient response applications now persist cross-turn state explicitly with diff --git a/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_orchestrator.py b/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_orchestrator.py index 9e779c8111a6..5f87013cc8e1 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_orchestrator.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_orchestrator.py @@ -73,23 +73,6 @@ logger = logging.getLogger("azure.ai.agentserver") -def _is_hosted_environment() -> bool: - """Return whether the agent is running in a Foundry-hosted container. - - Uses the canonical :class:`~azure.ai.agentserver.core.AgentConfig` derivation - (the same public API ``_routing`` already uses for Foundry auto-activation). - In a hosted deployment the resilient-task subsystem is auto-initialized with - no opt-out, so a missing TaskManager is a platform-infrastructure failure - rather than a reason to silently run a response non-durably. - - :return: ``True`` if running in a Foundry-hosted environment. - :rtype: bool - """ - from azure.ai.agentserver.core import AgentConfig # pylint: disable=import-outside-toplevel - - return AgentConfig.from_env().is_hosted - - _STORAGE_ERROR_MESSAGE = ( "An internal error occurred while storing the response. " "Subsequent retrieval is not guaranteed. Please retry the request." @@ -3925,13 +3908,16 @@ async def _start_resilient_background( Two outcomes when the resilient start cannot proceed: - - **No task subsystem installed** (the start raises - :class:`~azure.ai.agentserver.core.tasks.TaskManagerNotInitialized` — - e.g. an in-process test client whose lifespan never ran). When NOT - hosted, run the handler in-process via ``fallback_runner`` — there is - nothing to recover, so this is the legitimate non-durable path, NOT a - failure. When hosted, the subsystem is auto-initialized with no - opt-out, so its absence is a platform failure and is re-raised tagged. + - **No manager installed** — the start raises + :class:`~azure.ai.agentserver.core.tasks.TaskManagerNotInitialized`. + Because ``AgentServerHost`` fails the lifespan when resilient tasks are + ENABLED but construction/startup fails, a missing manager in a running + deployment means resilient tasks are DISABLED (opt-out) — durability + was not requested (this also covers in-process test clients whose + lifespan never ran). The signal is **swallowed** and the handler runs + in-process via ``fallback_runner`` — the response still executes and + persists (GET works); it is simply not crash-recoverable. NOT a + failure. - **Subsystem present but the start fails**: fail immediately. The exception is tagged as a platform infrastructure error and re-raised (no silent degradation to a non-durable task — that would hide a real @@ -3942,7 +3928,7 @@ async def _start_resilient_background( :param record: The mutable execution record. :type record: ResponseExecution :param fallback_runner: The shielded runner coroutine function to run - in-process when no task subsystem is installed. + in-process when resilient tasks are disabled. :type fallback_runner: Any :keyword disposition: One of ``"re-invoke"`` (Row 1: resilient_bg+bg+store — task body re-runs handler on recovery) or ``"mark-failed"`` @@ -3951,11 +3937,12 @@ async def _start_resilient_background( recovery). Stamped into task framework metadata so recovery dispatch can route without re-deriving the gate from request params. :paramtype disposition: str - :raises Exception: If durability is required but unavailable — either the - task subsystem is present and the resilient start fails, or the - deployment is hosted and the subsystem is missing. The exception is + :raises Exception: If the task subsystem is present and the resilient + start fails (e.g. the task-store write is rejected). The exception is tagged with ``PLATFORM_ERROR_TAG`` so the endpoint surfaces - ``x-platform-error-source: platform``. + ``x-platform-error-source: platform``. A missing subsystem + (``TaskManagerNotInitialized``) is NOT raised — it is swallowed and + handled via the in-process fallback (see above). """ from ._resilient_orchestrator import ( ResilientResponseOrchestrator, @@ -4004,27 +3991,22 @@ async def _start_resilient_background( # `previous_response_id`. Propagate so the endpoint layer # surfaces HTTP 409 `conversation_fork_not_supported`. raise - except TaskManagerNotInitialized as exc: - # No resilient-task subsystem is installed in this process. - if _is_hosted_environment(): - # Hosted deployments auto-initialize the subsystem with no - # opt-out, so its absence means initialization failed at boot - # (e.g. a misconfigured backend or a missing dependency). - # Durability is mandatory in production — fail loudly as a - # platform error rather than silently degrading a store=true - # response to a non-durable, connection-scoped task. - logger.error( - "Resilient task subsystem missing in hosted environment for response %s; failing the request", - ctx.response_id, - ) - setattr(exc, PLATFORM_ERROR_TAG, True) - await self._runtime_state.delete(ctx.response_id) - raise - # Non-hosted (local dev, or unit/contract tests whose ASGI lifespan - # never ran). Nothing to recover — run the handler in-process. This - # is the legitimate non-durable path, NOT a failure. (When a manager - # IS present — hosted, or a local file-provider deployment — the - # start above succeeds and we use the resilient task.) + except TaskManagerNotInitialized: + # No manager is installed. Because ``AgentServerHost`` fails the + # lifespan when resilient tasks are ENABLED but construction/startup + # fails, a missing manager in a running deployment means resilient + # tasks are simply DISABLED (opt-out) — recovery/durability was not + # requested. (It also covers in-process test clients whose lifespan + # never ran.) SWALLOW and run the handler in-process: the response + # still executes and persists (GET works), it is simply not + # crash-recoverable. This is the deliberate non-durable path, NOT a + # failure. + logger.info( + "Resilient task subsystem not enabled for response %s; running handler " + "in-process (non-durable). Enable via set_resilient_tasks_enabled(True) " + "(or resilient_background) for crash recovery.", + ctx.response_id, + ) record.execution_task = asyncio.create_task(fallback_runner()) except Exception as exc: # pylint: disable=broad-exception-caught # The resilient-task subsystem IS present but starting the task diff --git a/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_resilient_orchestrator.py b/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_resilient_orchestrator.py index 72b1e2dc9061..32b34f6f6fc9 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_resilient_orchestrator.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_resilient_orchestrator.py @@ -427,6 +427,32 @@ def __init__( # function and does not need this reference. self._parent_orchestrator = parent_orchestrator + # Opting into resilient background responses implies the durable task + # subsystem must be constructed (and recovery enabled) for this + # deployment. The subsystem is gated SOLELY on the process-global switch + # (``AgentServerHost`` constructs the ``TaskManager`` only when it is + # set), so translate the explicit ``resilient_background`` opt-in into + # that switch here, at host construction time (before the ASGI lifespan + # runs). + # + # NOTE: only ``resilient_background`` auto-enables the subsystem — + # ``steerable_conversations`` intentionally does NOT. Recovery is tied to + # ``resilient_background`` alone; a steerable host that wants durability + # must set ``resilient_background=True`` (or call + # ``set_resilient_tasks_enabled(True)`` explicitly). The two-switch UX is + # a known rough edge to smooth over post-Public-Preview. + # + # A host that leaves ``resilient_background`` off (and does not set the + # switch) constructs no ``TaskManager``: ``store=true`` work degrades to + # non-durable in-process execution (``_start_resilient_background`` + # swallows ``TaskManagerNotInitialized``). + if options.resilient_background: + from azure.ai.agentserver.core.tasks import ( # pylint: disable=import-outside-toplevel + set_resilient_tasks_enabled, + ) + + set_resilient_tasks_enabled(True) + # Spec 023 — per-request primitive dispatch (SOT §6.6). # Two task primitives are registered per deployment; ``_pick_primitive`` # selects per request based on (conversation_id, previous_response_id, @@ -1220,7 +1246,17 @@ async def start_resilient( # auto-queues against an in-flight chain and returns a TaskRun # whose ``is_queued`` is True (the public-surface detection signal). # See the queued-vs-fresh check below. - task_run = await picked_primitive.start(**start_kwargs) + try: + task_run = await picked_primitive.start(**start_kwargs) + except BaseException: + # If the primitive never started (e.g. ``TaskManagerNotInitialized`` + # when resilient tasks are disabled, or any start failure), the + # resilient task body — whose ``finally`` normally evicts the + # out-of-band refs — never runs. Drop the cache entry here so we do + # not permanently retain the record/context/parsed-request/cancel + # event for a response that fell back to in-process execution. + _RUNTIME_REFS.pop(response_id, None) + raise # Store the task run reference on the record for observability record.resilient_task_run = task_run # type: ignore[attr-defined] diff --git a/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_routing.py b/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_routing.py index d0fb1a76e257..c35257812fd2 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_routing.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/azure/ai/agentserver/responses/hosting/_routing.py @@ -175,6 +175,51 @@ def _configure_streams_registry(runtime_options: ResponsesServerOptions) -> None ) +def _log_startup_configuration( + resolved_provider: "ResponseProviderProtocol", runtime_options: ResponsesServerOptions +) -> None: + """Log the responses startup configuration and durability mode. + + Emitted once at host construction. The resilience line reflects the final + resolved state (the resilient orchestrator auto-enables the switch when + ``resilient_background`` is set). When durability is disabled we log a + WARNING so operators are not surprised that a ``store=true`` response killed + mid-flight by an ungraceful crash stays ``in_progress`` on a later GET (no + crash recovery) — matching a plain stateless server. + + :param resolved_provider: The resolved response persistence provider. + :type resolved_provider: ResponseProviderProtocol + :param runtime_options: The responses server options. + :type runtime_options: ResponsesServerOptions + """ + logger.info( + "Responses protocol: storage_provider=%s, default_model=%s, " + "default_fetch_history_count=%s, shutdown_grace_period=%ss", + type(resolved_provider).__name__, + runtime_options.default_model or "(not set)", + runtime_options.default_fetch_history_count, + runtime_options.shutdown_grace_period_seconds, + ) + + from azure.ai.agentserver.core.tasks import ( # pylint: disable=import-outside-toplevel + resilient_tasks_enabled, + ) + + if resilient_tasks_enabled(): + logger.info( + "Responses resilience: ENABLED - store=true responses run inside durable " + "tasks with crash recovery (in-flight responses are recovered/marked failed " + "on restart)." + ) + else: + logger.warning( + "Responses resilience: DISABLED - store=true responses run in-process and are " + "NOT durable across an ungraceful crash: a response in-flight when the process " + "is hard-killed stays in_progress on a later GET (no mark-failed/recovery). " + "Enable durability via resilient_background, or set_resilient_tasks_enabled(True)." + ) + + def _validate_handler_signature(fn: Any) -> None: """Reject sync handlers and 2-arg signatures. @@ -486,15 +531,8 @@ def __init__( # Stash endpoint reference for request_shutdown() access. self._endpoint = endpoint - # --- Responses startup configuration logging --- - logger.info( - "Responses protocol: storage_provider=%s, default_model=%s, " - "default_fetch_history_count=%s, shutdown_grace_period=%ss", - type(resolved_provider).__name__, - runtime_options.default_model or "(not set)", - runtime_options.default_fetch_history_count, - runtime_options.shutdown_grace_period_seconds, - ) + # --- Responses startup configuration + durability-mode logging --- + _log_startup_configuration(resolved_provider, runtime_options) # ------------------------------------------------------------------ # Shutdown notification diff --git a/sdk/agentserver/azure-ai-agentserver-responses/tests/conftest.py b/sdk/agentserver/azure-ai-agentserver-responses/tests/conftest.py index b88f18a98698..84521fa950ae 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/tests/conftest.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/tests/conftest.py @@ -30,6 +30,38 @@ def pytest_configure(config): ) +@pytest.fixture(autouse=True) +def _reset_resilient_tasks_global_state(): + """Contain process-global resilient-tasks state per test. + + The resilient-tasks enable switch and the ``TaskManager`` singleton are + process-global. A test that enables resilient tasks — e.g. constructs a + ``resilient_background=True`` host, which auto-enables the switch, or calls + ``set_resilient_tasks_enabled(True)`` — would otherwise leak that state into + subsequent tests in the same pytest process (a plain host would then build a + manager and behave durably). Snapshot both at test start and restore at the + end so each test is isolated. + """ + from azure.ai.agentserver.core.tasks import ( # pylint: disable=import-outside-toplevel + _manager as _mgr_mod, + ) + from azure.ai.agentserver.core.tasks import ( # pylint: disable=import-outside-toplevel + resilient_tasks_enabled, + set_resilient_tasks_enabled, + ) + from azure.ai.agentserver.core.tasks._manager import ( # pylint: disable=import-outside-toplevel + set_task_manager, + ) + + saved_flag = resilient_tasks_enabled() + saved_mgr = _mgr_mod._manager # noqa: SLF001 # pylint: disable=protected-access + try: + yield + finally: + set_task_manager(saved_mgr) + set_resilient_tasks_enabled(saved_flag) + + @pytest.fixture(autouse=True) def _isolated_resilient_tasks_root(tmp_path): """Isolate the LocalFileTaskProvider's default storage per test. diff --git a/sdk/agentserver/azure-ai-agentserver-responses/tests/contract/test_resilient_start_failure.py b/sdk/agentserver/azure-ai-agentserver-responses/tests/contract/test_resilient_start_failure.py index 12bcf8f93728..7a71295dc280 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/tests/contract/test_resilient_start_failure.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/tests/contract/test_resilient_start_failure.py @@ -1,16 +1,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Contract tests: a resilient task-start failure must FAIL the request. +"""Contract tests: resilient task-start behavior. -When the resilient-task subsystem IS installed (hosted) but starting the -task-backed background execution fails, the server must NOT silently degrade -to a non-durable, connection-scoped ``asyncio.create_task`` (which loses crash -recovery while looking healthy). Instead it must fail immediately and surface -the failure as a *platform* error source — exactly like a Foundry storage -failure. +Two distinct cases with DIFFERENT contracts: -The legitimate "no task subsystem at all" case (e.g. a test client without a -TaskManager) must STILL run the handler in-process — that is not a failure. +1. The resilient-task subsystem IS installed but starting the task-backed + execution *fails* (the start call raises): the server must NOT silently + degrade to a non-durable ``asyncio.create_task`` — it must fail immediately + and surface a *platform* error source (like a Foundry storage failure). A + real durability failure must not hide behind a healthy-looking response. + +2. No task subsystem is installed at all (the host did not enable resilient + tasks via ``set_resilient_tasks_enabled``): this is the deliberate opt-out + path. ``TaskManagerNotInitialized`` is SWALLOWED and the handler runs + in-process — the response still executes and persists (GET works), it is + simply not crash-recoverable. This applies regardless of hosted vs local. """ from __future__ import annotations @@ -23,12 +27,40 @@ from starlette.testclient import TestClient from azure.ai.agentserver.core._platform_headers import ERROR_DETAIL, ERROR_SOURCE +from azure.ai.agentserver.core.tasks import ( + TaskManagerNotInitialized, + resilient_tasks_enabled, + set_resilient_tasks_enabled, +) +from azure.ai.agentserver.core.tasks._manager import get_task_manager, set_task_manager from azure.ai.agentserver.responses import ResponsesAgentServerHost -from azure.ai.agentserver.responses.hosting import _orchestrator as _orch from azure.ai.agentserver.responses.hosting import _resilient_orchestrator as _ro from azure.ai.agentserver.responses.streaming._event_stream import ResponseEventStream +@pytest.fixture() +def _switch_off() -> Any: + """Ensure the process-global resilient-tasks state is OFF for the test. + + Both the enable switch and the ``TaskManager`` singleton are process-global + and other tests in a shared pytest process may have flipped the switch on or + installed a manager (e.g. by constructing a ``resilient_background=True`` + host). Snapshot, reset both to the switch-off / no-manager state so the + production opt-out path is genuinely exercised, then restore afterwards. + """ + from azure.ai.agentserver.core.tasks import _manager as _mgr_mod # pylint: disable=import-outside-toplevel + + saved_flag = resilient_tasks_enabled() + saved_mgr = _mgr_mod._manager # noqa: SLF001 # pylint: disable=protected-access + set_resilient_tasks_enabled(False) + set_task_manager(None) + try: + yield + finally: + set_task_manager(saved_mgr) + set_resilient_tasks_enabled(saved_flag) + + async def _noop_handler(request: Any, context: Any, cancellation_signal: asyncio.Event) -> AsyncIterator[Any]: async def _events() -> AsyncIterator[Any]: stream = ResponseEventStream(response_id=context.response_id, model=getattr(request, "model", None) or "") @@ -114,12 +146,12 @@ def test_streaming_start_failure_emits_standalone_error_event(self, _start_fails class TestNoTaskManagerStillRunsHandler: - """Regression: no task subsystem (non-hosted) → handler runs in-process.""" + """No task subsystem (opt-out) → handler runs in-process (non-durable).""" def test_no_manager_background_runs_handler_ok(self) -> None: # Plain TestClient (no `with` → no lifespan → no TaskManager installed) - # and non-hosted (FOUNDRY_HOSTING_ENVIRONMENT unset) → legitimate - # in-process fallback, NOT a failure. + # → the outer catch swallows TaskManagerNotInitialized and runs the + # handler in-process. Legitimate opt-out path, NOT a failure. client = _build_client() resp = client.post( "/responses", @@ -129,36 +161,62 @@ def test_no_manager_background_runs_handler_ok(self) -> None: assert ERROR_SOURCE not in resp.headers -class TestHostedNoTaskManagerFailsLoudly: - """Hosted + no manager → durability is mandatory → fail as platform error. +class TestNoTaskManagerSwallowsAndRunsInProcess: + """Recovery is opt-in: when no ``TaskManager`` is installed, ``store=true`` + work is NOT failed as a platform error — the responses outer catch swallows + ``TaskManagerNotInitialized`` and runs the handler in-process (non-durable). + The response still executes AND persists (GET works). - In a hosted deployment the resilient-task subsystem is auto-initialized - with no opt-out, so its absence is a platform-infrastructure failure — the - server must NOT silently degrade a ``store=true`` response to a non-durable - in-process run. - """ + These tests exercise the responses **swallow** behavior specifically. The + no-manager condition is forced deterministically via ``set_task_manager(None)`` + inside the running lifespan so the test does not depend on process-global + state left by other tests, nor on which core build performs (or skips) the + manager construction — that gating is covered by the core opt-in tests.""" - def test_hosted_no_manager_background_fails_platform_500(self, monkeypatch: pytest.MonkeyPatch) -> None: - # Simulate hosted at the gate (no real env change → no Foundry store - # auto-activation) while the bare TestClient has no manager installed. - monkeypatch.setattr(_orch, "_is_hosted_environment", lambda: True) - client = _build_client() - resp = client.post( - "/responses", - json={"model": "test", "input": "hi", "stream": False, "store": True, "background": True}, - ) - assert resp.status_code == 500, resp.text - assert resp.headers.get(ERROR_SOURCE) == "platform" - assert ERROR_DETAIL in resp.headers - assert "in_progress" not in resp.text + def test_switch_off_no_manager_installed_and_response_completes(self, _switch_off: Any) -> None: + with _build_client() as client: + # Force the switch-off condition deterministically: whatever the + # lifespan installed, ensure no manager is present so the responses + # swallow path is the one under test. + set_task_manager(None) + with pytest.raises(TaskManagerNotInitialized): + get_task_manager() - def test_hosted_no_manager_streaming_emits_error_event(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(_orch, "_is_hosted_environment", lambda: True) - client = _build_client() - resp = client.post( - "/responses", - json={"model": "test", "input": "hi", "stream": True, "store": True, "background": True}, - ) - types = [e["type"] for e in _collect_sse_events(resp.text)] - assert "error" in types, f"expected a standalone error event, got: {types}" - assert "response.completed" not in types, f"must not complete, got: {types}" + resp = client.post( + "/responses", + json={"model": "test", "input": "hi", "stream": False, "store": True, "background": True}, + ) + assert resp.status_code == 200, resp.text + # Swallowed → not a platform error. + assert ERROR_SOURCE not in resp.headers + response_id = resp.json()["id"] + + # The in-process fallback runs AND persists: GET reaches a terminal. + import time + + deadline = time.monotonic() + 10.0 + status = None + while time.monotonic() < deadline: + got = client.get(f"/responses/{response_id}") + if got.status_code == 200: + status = got.json().get("status") + if status in ("completed", "failed", "cancelled"): + break + time.sleep(0.05) + assert status == "completed", f"expected completed via in-process fallback, got {status}" + + def test_switch_off_no_manager_streaming_runs_in_process(self, _switch_off: Any) -> None: + with _build_client() as client: + # Force the switch-off condition deterministically (see above). + set_task_manager(None) + with pytest.raises(TaskManagerNotInitialized): + get_task_manager() + + resp = client.post( + "/responses", + json={"model": "test", "input": "hi", "stream": True, "store": True, "background": True}, + ) + types = [e["type"] for e in _collect_sse_events(resp.text)] + # In-process fallback runs the handler to completion; no error surface. + assert "error" not in types, f"must not surface an error on opt-out, got: {types}" + assert "response.completed" in types, f"expected completion, got: {types}" diff --git a/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_steering_handler.py b/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_steering_handler.py index b8ae32eb4051..d5f68eda85e3 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_steering_handler.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_steering_handler.py @@ -41,6 +41,7 @@ import asyncio import os +from azure.ai.agentserver.core.tasks import set_resilient_tasks_enabled from azure.ai.agentserver.responses import ( CreateResponse, ResponseContext, @@ -77,6 +78,11 @@ def _env_int(name: str, default: int) -> int: steerable_conversations=True, shutdown_grace_period_seconds=_SHUTDOWN_GRACE_S, ) +# Steering (mid-turn input queuing) is implemented by the multi-turn task +# primitive and needs the TaskManager. Since only ``resilient_background`` now +# auto-enables the subsystem (steerable does not), enable it explicitly so the +# steering conformance is valid even when CONFORMANCE_RESILIENT_BACKGROUND=false. +set_resilient_tasks_enabled(True) app = ResponsesAgentServerHost(options=options) _turn_counts: dict[str, int] = {} diff --git a/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_test_handler.py b/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_test_handler.py index b528eca60fb8..e7d796589285 100644 --- a/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_test_handler.py +++ b/sdk/agentserver/azure-ai-agentserver-responses/tests/e2e/resilience_contract/_test_handler.py @@ -53,6 +53,7 @@ import asyncio import os +from azure.ai.agentserver.core.tasks import set_resilient_tasks_enabled from azure.ai.agentserver.responses import ( CreateResponse, ResponseContext, @@ -101,6 +102,12 @@ def _env_int(name: str, default: int) -> int: resilient_background=_RESILIENT_BG, shutdown_grace_period_seconds=_SHUTDOWN_GRACE_S, ) +# Conformance exercises the durable-response subsystem (Rows 1/2/3), which is +# gated on the resilient-tasks switch. Row 2 runs with resilient_background=False +# (which does NOT auto-enable the switch), so enable it explicitly here — the +# server must construct the TaskManager and run recovery to honour the Row 2 +# mark-failed-on-crash contract. +set_resilient_tasks_enabled(True) app = ResponsesAgentServerHost(options=options)