diff --git a/ts/.prettierignore b/ts/.prettierignore index a66db064b..301534e59 100644 --- a/ts/.prettierignore +++ b/ts/.prettierignore @@ -43,6 +43,11 @@ examples/websiteAliases/cache docs/overview/action-browser.html docs/overview/action-browser.json +# Generated translation-test reliability doc (regenerated by +# reliability/report.ts; not hand-edited, so exclude to avoid table-alignment +# churn with the generator's own output). +packages/defaultAgentProvider/test/reliability/README.md + # VS Studio Files .vs/** diff --git a/ts/packages/agents/player/README.AUTOGEN.md b/ts/packages/agents/player/README.AUTOGEN.md index 891e9d88c..df0284404 100644 --- a/ts/packages/agents/player/README.AUTOGEN.md +++ b/ts/packages/agents/player/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # music — AI-generated documentation @@ -16,11 +16,11 @@ The `music` package, located in `ts/packages/agents/player/`, is a TypeAgent app ## What it does -The `music` package provides a wide range of actions for interacting with Spotify. These actions are organized into several functional categories: +The `music` package provides a comprehensive set of actions for interacting with Spotify. These actions are grouped into several functional categories: ### Playback Control -The agent allows users to control playback on active Spotify devices. Key actions include: +The agent supports controlling playback on active Spotify devices. Key actions include: - `playTrack`, `pause`, `resume`, `next`, `previous`: Basic playback controls. - `setVolume`, `changeVolume`: Adjust playback volume. @@ -36,7 +36,7 @@ Users can manage their playlists with actions such as: ### Search and Discovery -The agent supports searching Spotify's music catalog with actions like: +The agent allows users to search Spotify's music catalog with actions like: - `searchTracks`, `searchForPlaylists`, `searchArtists`, `searchAlbums`, `searchGenres`: Search for tracks, playlists, artists, albums, or genres. @@ -164,6 +164,6 @@ _3 environment variables referenced from `./src/` (set in `ts/.env` or your shel --- -_Auto-generated against commit `493ac5696974a3cab9c5528399a245063e7eb727` on `2026-07-18T23:55:00.731Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter music docs:verify-links` to spot-check._ +_Auto-generated against commit `8192acc149ab6f52296dcc71202b20253b6a3089` on `2026-07-22T05:16:33.340Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter music docs:verify-links` to spot-check._ diff --git a/ts/packages/agents/player/src/agent/playerManifest.json b/ts/packages/agents/player/src/agent/playerManifest.json index d72f887f4..7f1b131f7 100644 --- a/ts/packages/agents/player/src/agent/playerManifest.json +++ b/ts/packages/agents/player/src/agent/playerManifest.json @@ -2,7 +2,7 @@ "emojiChar": "🎧", "description": "Agent to play music", "schema": { - "description": "Music Player agent that lets you search for, play and control music.", + "description": "Music Player agent to play and control music. Use it to play, start, put on, queue, or listen to a specific song, artist, album, genre, or playlist (named or described by mood, e.g. 'chill music for studying'); control playback (pause, resume, skip/next, previous, shuffle, repeat, volume); choose the playback device; and browse, search, or manage playlists and favorites.", "originalSchemaFile": "./playerSchema.ts", "schemaFile": "../../dist/agent/playerSchema.pas.json", "grammarFile": "../../dist/agent/playerSchema.ag.json", diff --git a/ts/packages/aiclient/README.AUTOGEN.md b/ts/packages/aiclient/README.AUTOGEN.md index dba398804..bdaacc5fb 100644 --- a/ts/packages/aiclient/README.AUTOGEN.md +++ b/ts/packages/aiclient/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/aiclient — AI-generated documentation @@ -12,38 +12,40 @@ ## Overview -The `@typeagent/aiclient` package is a TypeScript library designed to interact with AI APIs, including OpenAI endpoints (both Azure-hosted and OpenAI-hosted) and Bing search APIs. It is primarily used within the TypeAgent project for sample agents and examples, providing tools to manage API settings, authentication, and multi-region endpoint pools. +The `@typeagent/aiclient` package is a TypeScript library designed to interact with AI APIs, including OpenAI (both Azure-hosted and OpenAI-hosted) and Bing. It is primarily used within the TypeAgent project for sample agents and examples. The library provides tools for API settings management, authentication, and multi-region endpoint pooling to ensure high availability and resilience. ## What it does -The `aiclient` package provides a range of capabilities for interacting with AI services: +The `aiclient` package offers the following capabilities: -- **OpenAI API Support**: Includes functionality for accessing OpenAI models hosted on Azure and OpenAI, such as GPT-4 and embedding models. -- **Bing API Support**: Provides tools for interacting with Bing search APIs. -- **Multi-region Endpoint Pools**: Implements a mechanism to distribute requests across multiple endpoints, mitigating throttling and ensuring high availability. -- **Environment-based Configuration**: Reads settings such as API keys and endpoints from environment variables, simplifying integration with existing infrastructure. +- **OpenAI API Integration**: Supports GPT-4, embeddings, and image generation models hosted on Azure and OpenAI. +- **Bing API Integration**: Provides tools for querying Bing search APIs. +- **Multi-region Endpoint Pools**: Implements a mechanism to distribute requests across multiple endpoints, handling throttling and regional outages. +- **Environment-based Configuration**: Automatically discovers API settings, such as endpoints and API keys, from environment variables. - **Authentication**: Supports Azure identity-based authentication and API key-based authentication. +- **Debug Logging**: Offers detailed logging for endpoint selection, rotation, and retry behavior. ### Multi-region Endpoint Pools -The package's endpoint pool mechanism is designed to handle throttling and regional failures gracefully. It rotates requests among endpoints based on priority tiers and health status. Key features include: +The package's endpoint pooling mechanism ensures reliability and performance by: -- **Priority-based Selection**: Endpoints are grouped into priority tiers, with the lowest-priority tier containing healthy endpoints being selected. -- **Cooldown Mechanism**: Endpoints that return transient errors (e.g., 429, 5xx) are temporarily marked as unavailable and excluded from selection. -- **Custom Pool Configuration**: Users can override the default endpoint discovery and priority settings using environment variables. +- **Endpoint Discovery**: Scanning environment variables to build pools of endpoints for each model and region. +- **Priority-based Selection**: Grouping endpoints into priority tiers and selecting the lowest-priority tier with healthy endpoints. +- **Cooldown Mechanism**: Temporarily marking endpoints as unavailable after transient errors (e.g., 429, 5xx). +- **Custom Pool Configuration**: Allowing users to override default endpoint discovery and priority settings using JSON-based environment variables. ### Debugging and Logging -The package includes debug logging capabilities for monitoring endpoint selection, rotation, and retry behavior. Enabling the `typeagent:pool` namespace provides detailed logs for troubleshooting. +Debug logging can be enabled for monitoring endpoint selection, rotation, and retry behavior. Use the `typeagent:pool` namespace for detailed logs. ## Setup -To use the `@typeagent/aiclient` package, you need to configure the following environment variables: +To use the `@typeagent/aiclient` package, configure the following environment variables: -- **`COPILOT_ALLOW_GET_PROVIDER_ENDPOINT`**: Controls whether the Copilot SDK is allowed to retrieve provider endpoints dynamically. +- **`COPILOT_ALLOW_GET_PROVIDER_ENDPOINT`**: Determines whether the Copilot SDK can dynamically retrieve provider endpoints. - **`TYPEAGENT_COPILOT_SDK_LOG_LEVEL`**: Sets the log level for the Copilot SDK. Valid values include `none`, `error`, `warning`, `info`, `debug`, and `all`. -Additionally, the package relies on a set of environment variables for endpoint pools and API keys. These are dynamically discovered based on the model and region. For example: +Additionally, the package relies on dynamically discovered environment variables for endpoint pools and API keys. Examples include: - `AZURE_OPENAI_ENDPOINT__`: Specifies the endpoint for a given model and region. - `AZURE_OPENAI_API_KEY__`: Specifies the API key for a given model and region. @@ -142,6 +144,6 @@ _2 environment variables referenced from `./src/` (set in `ts/.env` or your shel --- -_Auto-generated against commit `5cbcf613f047f08749d0451296eb1cdc610ae414` on `2026-07-17T18:24:18.404Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/aiclient docs:verify-links` to spot-check._ +_Auto-generated against commit `3849e13a69115bff7c01ea3995c20bb2cf23343f` on `2026-07-22T05:45:45.787Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/aiclient docs:verify-links` to spot-check._ diff --git a/ts/packages/aiclient/src/restClient.ts b/ts/packages/aiclient/src/restClient.ts index 21abf4986..a5ecac6f2 100644 --- a/ts/packages/aiclient/src/restClient.ts +++ b/ts/packages/aiclient/src/restClient.ts @@ -600,6 +600,7 @@ export type PoolCallOptions = { signal?: AbortSignal | undefined; }; +// code-complexity-allow: endpoint-pool retry/cooldown flow needs explicit per-status handling. async function fetchWithPool( pool: EndpointPool, method: "POST" | "GET", @@ -741,6 +742,23 @@ async function fetchWithPool( continue; } + // callFetch can resolve to `undefined` (not only throw) on a dropped + // connection; the single-endpoint fetchWithRetry path guards this + // explicitly (throwing "fetch: No response"). Without the same guard + // here, `response.status` below would throw a raw TypeError ("Cannot + // read properties of undefined (reading 'status')") that + // isTransientRequestError does NOT match, so it would surface as a hard + // failure instead of rotating to the next pool member. + // NOTE: this is the suspected (not yet stack-confirmed) source of a rare + // intermittent "reading 'status'" failure seen in the translation + // reliability tests. If that error recurs, log `response`/`.stack` here + // to confirm this is the site. + if (response === undefined) { + markTransientFailure(member); + lastError = error("fetch error: No response"); + continue; + } + if (response.status === 200 || response.status === 201) { markSuccess(member); return success(response); diff --git a/ts/packages/cache/README.AUTOGEN.md b/ts/packages/cache/README.AUTOGEN.md index 880698b9c..081dc14b0 100644 --- a/ts/packages/cache/README.AUTOGEN.md +++ b/ts/packages/cache/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # agent-cache — AI-generated documentation @@ -12,77 +12,84 @@ ## Overview -The `agent-cache` package is a TypeScript library that provides a construction cache for TypeAgent requests. It is designed to optimize the processing of user requests by caching parsed grammar and rules, referred to as "constructions." These constructions enable local translation of user requests, reducing the need for repeated interactions with external Large Language Models (LLMs). This approach improves system performance and reduces latency. +The `agent-cache` package is a TypeScript library that provides a construction cache for handling TypeAgent requests. It is designed to optimize the processing of user requests by caching parsed grammar and rules, referred to as "constructions." These constructions enable the system to perform local translations of user requests, bypassing the need for repeated interactions with external Large Language Models (LLMs). This reduces latency and improves overall system efficiency. ## What it does -The `agent-cache` package focuses on enhancing the efficiency of request handling in the TypeAgent ecosystem. Its key functionalities include: +The `agent-cache` package plays a critical role in the TypeAgent ecosystem by managing the explanation and caching of user request transformations. Its primary capabilities include: -- **Request Explanation**: Using the `explainRequest` action, the package explains how a user request is transformed into an action. This explanation is then used to generate constructions. -- **Construction Caching**: The `cacheConstruction` action allows the system to store constructions derived from user requests. These constructions are reusable for similar requests, reducing the need for repeated LLM queries. -- **Local Request Translation**: Cached constructions are used to translate user requests locally, bypassing the LLM and improving response times. -- **Cache Management**: The package provides tools to load, store, and manage construction caches. For example, the `loadConstructionCacheFile` action enables loading pre-existing cache files for immediate use. -- **Support for Multiple Explainers**: The package supports multiple explainers, each with its own validator and construction creator. This allows for experimentation with different approaches to optimize the explanation and translation process. +- **Request Explanation**: The package uses actions like `explainRequest` to analyze and explain how user requests are transformed into actions. This explanation is then used to generate constructions. +- **Construction Caching**: Constructions derived from user requests are stored using actions like `cacheConstruction`. These cached constructions can be reused for similar requests, reducing the need for repeated LLM queries. +- **Local Request Translation**: Cached constructions allow the system to translate user requests locally, bypassing the LLM and improving response times. +- **Cache Management**: The package provides tools for managing construction caches, such as loading pre-existing cache files with the `loadConstructionCacheFile` action. +- **Support for Multiple Explainers**: The package supports multiple explainers, each with its own validator and construction creator. This enables experimentation with different explanation and translation approaches. + +By leveraging these features, `agent-cache` enhances the efficiency and scalability of the TypeAgent system. ## Setup -The `agent-cache` package requires minimal setup. To get started: - -1. Install the package and its dependencies using your package manager (e.g., `pnpm install`). -2. Ensure the following internal dependencies are available in your workspace: - - `@typeagent/action-grammar` - - `@typeagent/action-schema` - - `@typeagent/agent-sdk` - - `@typeagent/aiclient` - - `@typeagent/common-utils` - - `telemetry` - - `test-lib` - - `typechat-utils` -3. Ensure the following external dependencies are installed: - - `async` - - `chalk` - - `debug` - - `regexp.escape` - - `typechat` - -For additional setup instructions, refer to the hand-written README. +To use the `agent-cache` package, follow these steps: + +1. **Install Dependencies**: + + - Use your package manager (e.g., `pnpm install`) to install the package and its dependencies. + - Ensure the following internal dependencies are available in your workspace: + - `@typeagent/action-grammar` + - `@typeagent/action-schema` + - `@typeagent/agent-sdk` + - `@typeagent/aiclient` + - `@typeagent/common-utils` + - `telemetry` + - `test-lib` + - `typechat-utils` + - Ensure the following external dependencies are installed: + - `async` + - `chalk` + - `debug` + - `regexp.escape` + - `typechat` + +2. **Configuration**: + - No additional configuration is required for basic usage. For advanced scenarios, such as adding new explainers, refer to the "How to extend" section. + +For more detailed setup instructions, consult the hand-written README. ## Key Files -The `agent-cache` package is organized into several key modules, each with specific responsibilities. Below is an overview of the key files: +The `agent-cache` package is organized into several key modules, each responsible for specific aspects of cache management, explanation, and grammar handling. Below is an overview of the most important files: ### Cache Management -- [cache.ts](./src/cache/cache.ts): Core logic for managing the construction cache, including loading and storing constructions. -- [constructionStore.ts](./src/cache/constructionStore.ts): Handles the storage and retrieval of constructions. -- [explainWorkQueue.ts](./src/cache/explainWorkQueue.ts): Manages the queue for processing explanations. -- [factory.ts](./src/cache/factory.ts): Provides factory methods for creating explainers and caches. -- [grammarStore.ts](./src/cache/grammarStore.ts): Manages grammar storage and matching logic. -- [sortMatches.ts](./src/cache/sortMatches.ts): Implements sorting algorithms for matching results. -- [types.ts](./src/cache/types.ts): Defines types and interfaces for cache and grammar management. +- **[cache.ts](./src/cache/cache.ts)**: Implements the core logic for managing the construction cache, including loading, storing, and retrieving constructions. +- **[constructionStore.ts](./src/cache/constructionStore.ts)**: Handles the storage and retrieval of constructions, including file-based operations. +- **[explainWorkQueue.ts](./src/cache/explainWorkQueue.ts)**: Manages the queue for processing explanations, ensuring efficient handling of multiple requests. +- **[factory.ts](./src/cache/factory.ts)**: Provides factory methods for creating explainers and caches. +- **[grammarStore.ts](./src/cache/grammarStore.ts)**: Manages grammar storage and matching logic for constructions. +- **[sortMatches.ts](./src/cache/sortMatches.ts)**: Implements sorting algorithms for matching results. +- **[types.ts](./src/cache/types.ts)**: Defines types and interfaces for cache and grammar management. ### Explanation -- [explainerFactories.ts](./src/explanation/explainerFactories.ts): Maintains a registry of supported explainers and their configurations. -- [genericExplainer.ts](./src/explanation/genericExplainer.ts): Provides a generic implementation for explainers. -- [schemaInfoProvider.ts](./src/explanation/schemaInfoProvider.ts): Supplies schema-related information for explanations. +- **[explainerFactories.ts](./src/explanation/explainerFactories.ts)**: Maintains a registry of supported explainers and their configurations. +- **[genericExplainer.ts](./src/explanation/genericExplainer.ts)**: Provides a generic implementation for explainers, including validation and construction creation. +- **[schemaInfoProvider.ts](./src/explanation/schemaInfoProvider.ts)**: Supplies schema-related information for explanations. ### Constructions -- [constructionCache.ts](./src/constructions/constructionCache.ts): Defines the structure and management of constructions. -- [constructionJSONTypes.ts](./src/constructions/constructionJSONTypes.ts): Specifies JSON types for serializing and deserializing constructions. -- [matchPart.ts](./src/constructions/matchPart.ts): Handles matching logic for construction parts. +- **[constructionCache.ts](./src/constructions/constructionCache.ts)**: Defines the structure and management of constructions. +- **[constructionJSONTypes.ts](./src/constructions/constructionJSONTypes.ts)**: Specifies JSON types for serializing and deserializing constructions. +- **[matchPart.ts](./src/constructions/matchPart.ts)**: Handles matching logic for construction parts. ### Grammar -- [exportGrammar.ts](./src/grammar/exportGrammar.ts): Converts constructions to grammar for local processing. -- [grammarStore.ts](./src/cache/grammarStore.ts): Manages grammar storage and matching. +- **[exportGrammar.ts](./src/grammar/exportGrammar.ts)**: Converts constructions to grammar for local processing. +- **[grammarStore.ts](./src/cache/grammarStore.ts)**: Manages grammar storage and matching. ### Entry Points -- [index.ts](./src/index.ts): Main entry point, exporting core types and functionalities. -- [indexBrowser.ts](./src/indexBrowser.ts): Browser-specific entry point, exporting constructions and match parts. -- [indexGrammar.ts](./src/indexGrammar.ts): Grammar-specific entry point, exporting grammar conversion functionalities. +- **[index.ts](./src/index.ts)**: Main entry point, exporting core types and functionalities. +- **[indexBrowser.ts](./src/indexBrowser.ts)**: Browser-specific entry point, exporting constructions and match parts. +- **[indexGrammar.ts](./src/indexGrammar.ts)**: Grammar-specific entry point, exporting grammar conversion functionalities. ## How to extend @@ -124,9 +131,9 @@ By following these guidelines, you can effectively extend and customize the `age ### Entry points -- default → `./dist/index.js` _(not found on disk)_ -- default → `./dist/indexBrowser.js` _(not found on disk)_ -- `./grammar` → `./dist/indexGrammar.js` _(not found on disk)_ +- default → [./dist/index.js](./dist/index.js) +- default → [./dist/indexBrowser.js](./dist/indexBrowser.js) +- `./grammar` → [./dist/indexGrammar.js](./dist/indexGrammar.js) ### Dependencies @@ -172,6 +179,6 @@ External: `async`, `chalk`, `debug`, `regexp.escape`, `typechat` --- -_Auto-generated against commit `366aaf867a7e8e5d130b6c87a365516bab725269` on `2026-07-07T09:05:05.703Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter agent-cache docs:verify-links` to spot-check._ +_Auto-generated against commit `8192acc149ab6f52296dcc71202b20253b6a3089` on `2026-07-22T05:16:33.340Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter agent-cache docs:verify-links` to spot-check._ diff --git a/ts/packages/cache/src/explanation/requestAction.ts b/ts/packages/cache/src/explanation/requestAction.ts index 19a6a0d73..0dd1aa510 100644 --- a/ts/packages/cache/src/explanation/requestAction.ts +++ b/ts/packages/cache/src/explanation/requestAction.ts @@ -17,6 +17,10 @@ export type HistoryContext = { promptSections: PromptSection[]; entities: PromptEntity[]; additionalInstructions?: string[] | undefined; + // Structured action(s) executed in the recent chat history window, oldest + // first. Rendered into the translation prompt so the model can see which + // requests were already carried out and avoid re-issuing them. + actions?: AppAction[] | undefined; activityContext?: ActivityContext | undefined; }; diff --git a/ts/packages/defaultAgentProvider/test/data/translate-e2e.json b/ts/packages/defaultAgentProvider/test/data/translate-e2e.json index f3164328e..a5a425def 100644 --- a/ts/packages/defaultAgentProvider/test/data/translate-e2e.json +++ b/ts/packages/defaultAgentProvider/test/data/translate-e2e.json @@ -78,7 +78,8 @@ "expected": [ "browser.lookupAndAnswer.lookupAndAnswerInternet", "dispatcher.pendingRequestAction" - ] + ], + "extraActions": [{ "duplicateOfPrevious": true }] }, [ { diff --git a/ts/packages/defaultAgentProvider/test/data/translate-history-e2e.json b/ts/packages/defaultAgentProvider/test/data/translate-history-e2e.json index 52c4a9070..5777c0640 100644 --- a/ts/packages/defaultAgentProvider/test/data/translate-history-e2e.json +++ b/ts/packages/defaultAgentProvider/test/data/translate-history-e2e.json @@ -229,7 +229,12 @@ "type": ["list"], "facets": [{ "name": "items", "value": ["eggs"] }] } - ] + ], + "action": { + "schemaName": "list", + "actionName": "addItems", + "parameters": { "items": ["eggs"], "listName": "grocery" } + } } }, { diff --git a/ts/packages/defaultAgentProvider/test/data/translate-image-history-e2e.json b/ts/packages/defaultAgentProvider/test/data/translate-image-history-e2e.json index b2f08b9d6..20b77b076 100644 --- a/ts/packages/defaultAgentProvider/test/data/translate-image-history-e2e.json +++ b/ts/packages/defaultAgentProvider/test/data/translate-image-history-e2e.json @@ -41,21 +41,11 @@ }, { "request": "show me attachment__0.png", - "question": "show me attachment__0.png", "expected": { "schemaName": "chat", "actionName": "showImageFile", "parameters": { "files": ["attachment__0.png"] - }, - "entities": { - "files": [ - { - "name": "attachment__0.png", - "type": ["file", "image", "data"], - "sourceAppAgentName": "chat" - } - ] } } } diff --git a/ts/packages/defaultAgentProvider/test/reliability/README.md b/ts/packages/defaultAgentProvider/test/reliability/README.md new file mode 100644 index 000000000..98f1d79b4 --- /dev/null +++ b/ts/packages/defaultAgentProvider/test/reliability/README.md @@ -0,0 +1,99 @@ +# Translation test reliability + +Suite-wide reliability of the live translation-stability tests +(`packages/defaultAgentProvider/test/translate*.test.ts`), measured as the +**mean number of translation tokens between flaky failures, across all suites** +(MTTF-in-tokens; higher = more reliable). + +Auto-generated by `reliability/report.ts` (compiled to +`dist/test/reliability/report.js`) - do not edit by hand. + +## Latest run + +- Generated: 2026-07-22T04:27:25.000Z +- Model: gpt-4.1 (repeat=5, post-fix) +- Suites: 9 +- Attempts (all suites): 390 +- Translation tokens (all suites): 1,696,140 +- Mean tokens / attempt: 4,349 + +| Metric | Events | MTTF (tokens between) | +|---|--:|--:| +| Test failures (assertion red) | 0 | no events this run (≥ 1,696,140 tokens) | +| Outcome variations (model non-determinism) | 1 | 1,696,140 tokens | + +## Reliability trend + +Across the last 5 run(s) (of 5 recorded). Each +point is one suite run; higher MTTF-in-tokens = more reliable. GitHub renders +the charts inline; the table below carries the same data for other viewers. + +```mermaid +xychart-beta + title "Tokens between test failures (higher = more reliable)" + x-axis ["#1", "#2", "#3", "#4", "#5"] + y-axis "Tokens between events" 0 --> 7500000 + line [1644570, 821876, 441657, 6786251, 1696140] +``` + +```mermaid +xychart-beta + title "Tokens between outcome variations (higher = more reliable)" + x-axis ["#1", "#2", "#3", "#4", "#5"] + y-axis "Tokens between events" 0 --> 1900000 + line [548190, 410938, 441657, 848281, 1696140] +``` + +| # | Generated | Model | Attempts | Failures | Variations | Tokens | MTTF failures | MTTF variations | +|--:|---|---|--:|--:|--:|--:|--:|--:| +| 1 | 2026-07-22T00:19:57.187Z | gpt-4.1 | 390 | 1 | 3 | 1,644,570 | 1,644,570 | 548,190 | +| 2 | 2026-07-22T00:27:59.609Z | gpt-4.1 | 390 | 2 | 4 | 1,643,751 | 821,876 | 410,938 | +| 3 | 2026-07-22T01:30:39.770Z | gpt-4.1 (repeat=20) | 1,560 | 15 | 15 | 6,624,856 | 441,657 | 441,657 | +| 4 | 2026-07-22T03:01:30.412Z | gpt-4.1 (repeat=20, post-fix) | 1,560 | 0 | 8 | 6,786,251 | ≥ 6,786,251 | 848,281 | +| 5 | 2026-07-22T04:27:25.000Z | gpt-4.1 (repeat=5, post-fix) | 390 | 0 | 1 | 1,696,140 | ≥ 1,696,140 | 1,696,140 | + +## Per-suite (latest run) + +| Suite | Attempts | Failures | Variations | Tokens | +|---|--:|--:|--:|--:| +| translate (no history) | 60 | 0 | 1 | 201,345 | +| translate (w/history) | 35 | 0 | 0 | 144,300 | +| translate (w/implicit lookup) | 50 | 0 | 0 | 275,965 | +| translate browser (w/history) | 40 | 0 | 0 | 0 | +| translate conversation | 5 | 0 | 0 | 30,790 | +| translate gate | 60 | 0 | 0 | 98,168 | +| translate image request (w/history) | 10 | 0 | 0 | 50,409 | +| translate mcp filesystem | 30 | 0 | 0 | 182,422 | +| translate player (w/history) | 100 | 0 | 0 | 712,741 | + +## What the numbers mean + +- **Test failures** are attempts where the assertion went red (the suite's real + reliability - a flaky failure the CI would surface). +- **Outcome variations** are attempts whose action signature differed from the + modal signature for the same request. This is the leading indicator of + flakiness: it counts run-to-run model non-determinism even when the assertion + tolerated it (e.g. an `anyof` clarify or a `duplicateOfPrevious` extra action). +- Only the LLM **translation** step's tokens are counted; grammar/cache-resolved + attempts consume no translation tokens and are deterministic, so they neither + add tokens nor cause flaky failures. +- When a run has zero events, MTTF is unbounded; the charts and `≥` cells use + the run's token total as a lower bound. + +## Regenerate + +From `ts/`: + +```powershell +$env:TRANSLATION_RELIABILITY_DIR = "$PWD/tmp/reliability" +Remove-Item $env:TRANSLATION_RELIABILITY_DIR -Recurse -Force -ErrorAction SilentlyContinue +pnpm --filter default-agent-provider build +Push-Location packages/defaultAgentProvider +pnpm run jest-esm --testPathPattern=translate --forceExit +Pop-Location +node packages/defaultAgentProvider/dist/test/reliability/report.js tmp/reliability packages/defaultAgentProvider/test/reliability/README.md +Remove-Item Env:\TRANSLATION_RELIABILITY_DIR +``` + +Each run appends one point to `reliability/history.json` (kept in git) and +re-renders the charts above. diff --git a/ts/packages/defaultAgentProvider/test/reliability/history.json b/ts/packages/defaultAgentProvider/test/reliability/history.json new file mode 100644 index 000000000..0ad364fb2 --- /dev/null +++ b/ts/packages/defaultAgentProvider/test/reliability/history.json @@ -0,0 +1,47 @@ +[ + { + "generated": "2026-07-22T00:19:57.187Z", + "model": "gpt-4.1", + "suites": 9, + "attempts": 390, + "failures": 1, + "variations": 3, + "tokens": 1644570 + }, + { + "generated": "2026-07-22T00:27:59.609Z", + "model": "gpt-4.1", + "suites": 9, + "attempts": 390, + "failures": 2, + "variations": 4, + "tokens": 1643751 + }, + { + "generated": "2026-07-22T01:30:39.770Z", + "model": "gpt-4.1 (repeat=20)", + "suites": 9, + "attempts": 1560, + "failures": 15, + "variations": 15, + "tokens": 6624856 + }, + { + "generated": "2026-07-22T03:01:30.412Z", + "model": "gpt-4.1 (repeat=20, post-fix)", + "suites": 9, + "attempts": 1560, + "failures": 0, + "variations": 8, + "tokens": 6786251 + }, + { + "generated": "2026-07-22T04:27:25.000Z", + "model": "gpt-4.1 (repeat=5, post-fix)", + "suites": 9, + "attempts": 390, + "failures": 0, + "variations": 1, + "tokens": 1696140 + } +] diff --git a/ts/packages/defaultAgentProvider/test/reliability/report.ts b/ts/packages/defaultAgentProvider/test/reliability/report.ts new file mode 100644 index 000000000..19ce4e684 --- /dev/null +++ b/ts/packages/defaultAgentProvider/test/reliability/report.ts @@ -0,0 +1,270 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Aggregates the per-suite reliability tallies written by translateTestCommon.ts +// (when TRANSLATION_RELIABILITY_DIR is set) into a single suite-wide reliability +// doc: the mean number of translation tokens between flaky failures across all +// translation test suites. Higher = more reliable. +// +// Each run is also appended to reliability/history.json (kept in git) so the doc +// can chart the reliability trend over time. The charts are emitted as mermaid +// `xychart-beta` blocks, which GitHub renders inline in the markdown. +// +// Compiled to dist/test/reliability/report.js. +// Usage: node report.js [modelName] + +import fs from "node:fs"; +import path from "node:path"; + +// Per-suite tally written by translateTestCommon.ts (one JSON file per suite). +type SuiteTally = { + name: string; + attempts: number; + failures: number; + variations: number; + tokens: number; +}; + +// One recorded run, appended to history.json so the doc can chart the trend. +// Raw counts are stored (not derived MTTF values) so past points recompute +// consistently if the MTTF formula changes. +type HistoryEntry = { + generated: string; + model: string; + suites: number; + attempts: number; + failures: number; + variations: number; + tokens: number; +}; + +const HISTORY_FILE = "history.json"; +// Most-recent runs shown in the history table and trend charts. +const HISTORY_SHOWN = 25; + +function fmt(n: number): string { + return n.toLocaleString("en-US"); +} + +// Mean tokens between events. With zero events in a run MTTF is unbounded, so use +// the run's token total as a lower bound to keep the value finite and chartable. +function mttfTokens(tokens: number, events: number): number { + return events > 0 ? Math.round(tokens / events) : tokens; +} + +// Prose form of the MTTF, spelling out the zero-event lower-bound case. +function mttfLabel(tokens: number, events: number): string { + return events > 0 + ? `${fmt(Math.round(tokens / events))} tokens` + : `no events this run (\u2265 ${fmt(tokens)} tokens)`; +} + +// Compact table-cell form: the number, or "≥N" for the zero-event lower bound. +function mttfCell(tokens: number, events: number): string { + return events > 0 + ? fmt(Math.round(tokens / events)) + : `\u2265 ${fmt(tokens)}`; +} + +// A single-series mermaid line chart. GitHub renders `xychart-beta` fenced blocks +// inline, so the trend graphs directly in the markdown. xychart-beta has no +// legend, so each series is charted separately under its own title. +function lineChart( + title: string, + labels: string[], + series: number[], +): string[] { + const max = Math.max(1, ...series); + const top = Math.ceil((max * 1.1) / 100000) * 100000 || 100000; + return [ + "```mermaid", + "xychart-beta", + ` title "${title}"`, + ` x-axis [${labels.join(", ")}]`, + ` y-axis "Tokens between events" 0 --> ${top}`, + ` line [${series.join(", ")}]`, + "```", + ]; +} + +function main(): void { + const resultsDir = process.argv[2]; + const outDoc = process.argv[3]; + const modelName = process.argv[4]; + if (!resultsDir || !outDoc) { + console.error( + "Usage: node report.js [modelName]", + ); + process.exit(1); + } + + const files = fs.existsSync(resultsDir) + ? fs.readdirSync(resultsDir).filter((f) => f.endsWith(".json")) + : []; + if (files.length === 0) { + console.error( + `No tally JSON files in '${resultsDir}'. Run the translate suite ` + + `with TRANSLATION_RELIABILITY_DIR set to that directory first.`, + ); + process.exit(1); + } + + const suites: SuiteTally[] = files + .map( + (f) => + JSON.parse( + fs.readFileSync(path.join(resultsDir, f), "utf-8"), + ) as SuiteTally, + ) + .sort((a, b) => a.name.localeCompare(b.name)); + + let attempts = 0; + let failures = 0; + let variations = 0; + let tokens = 0; + for (const s of suites) { + attempts += s.attempts; + failures += s.failures; + variations += s.variations; + tokens += s.tokens; + } + const perAttempt = attempts > 0 ? Math.round(tokens / attempts) : 0; + const generated = new Date().toISOString(); + const model = modelName ?? "(default)"; + + // Append this run to the persisted history. + const historyPath = path.join(path.dirname(outDoc), HISTORY_FILE); + const history: HistoryEntry[] = fs.existsSync(historyPath) + ? (JSON.parse(fs.readFileSync(historyPath, "utf-8")) as HistoryEntry[]) + : []; + history.push({ + generated, + model, + suites: suites.length, + attempts, + failures, + variations, + tokens, + }); + fs.writeFileSync(historyPath, JSON.stringify(history, null, 2) + "\n"); + + // Chart/table the most recent runs; ordinals tie chart x-axis to table rows. + const recent = history.slice(-HISTORY_SHOWN); + const startOrdinal = history.length - recent.length + 1; + const labels = recent.map((_, i) => `"#${startOrdinal + i}"`); + const failSeries = recent.map((h) => mttfTokens(h.tokens, h.failures)); + const varSeries = recent.map((h) => mttfTokens(h.tokens, h.variations)); + + const lines: string[] = [ + "# Translation test reliability", + "", + "Suite-wide reliability of the live translation-stability tests", + "(`packages/defaultAgentProvider/test/translate*.test.ts`), measured as the", + "**mean number of translation tokens between flaky failures, across all suites**", + "(MTTF-in-tokens; higher = more reliable).", + "", + "Auto-generated by `reliability/report.ts` (compiled to", + "`dist/test/reliability/report.js`) - do not edit by hand.", + "", + "## Latest run", + "", + `- Generated: ${generated}`, + `- Model: ${model}`, + `- Suites: ${suites.length}`, + `- Attempts (all suites): ${fmt(attempts)}`, + `- Translation tokens (all suites): ${fmt(tokens)}`, + `- Mean tokens / attempt: ${fmt(perAttempt)}`, + "", + "| Metric | Events | MTTF (tokens between) |", + "|---|--:|--:|", + `| Test failures (assertion red) | ${failures} | ${mttfLabel(tokens, failures)} |`, + `| Outcome variations (model non-determinism) | ${variations} | ${mttfLabel(tokens, variations)} |`, + "", + "## Reliability trend", + "", + `Across the last ${recent.length} run(s) (of ${history.length} recorded). Each`, + "point is one suite run; higher MTTF-in-tokens = more reliable. GitHub renders", + "the charts inline; the table below carries the same data for other viewers.", + "", + ...lineChart( + "Tokens between test failures (higher = more reliable)", + labels, + failSeries, + ), + "", + ...lineChart( + "Tokens between outcome variations (higher = more reliable)", + labels, + varSeries, + ), + "", + "| # | Generated | Model | Attempts | Failures | Variations | Tokens | MTTF failures | MTTF variations |", + "|--:|---|---|--:|--:|--:|--:|--:|--:|", + ...recent.map((h, i) => { + const n = startOrdinal + i; + return ( + `| ${n} | ${h.generated} | ${h.model} | ${fmt(h.attempts)} | ` + + `${h.failures} | ${h.variations} | ${fmt(h.tokens)} | ` + + `${mttfCell(h.tokens, h.failures)} | ${mttfCell(h.tokens, h.variations)} |` + ); + }), + "", + "## Per-suite (latest run)", + "", + "| Suite | Attempts | Failures | Variations | Tokens |", + "|---|--:|--:|--:|--:|", + ...suites.map( + (s) => + `| ${s.name} | ${fmt(s.attempts)} | ${s.failures} | ${s.variations} | ${fmt(s.tokens)} |`, + ), + "", + "## What the numbers mean", + "", + "- **Test failures** are attempts where the assertion went red (the suite's real", + " reliability - a flaky failure the CI would surface).", + "- **Outcome variations** are attempts whose action signature differed from the", + " modal signature for the same request. This is the leading indicator of", + " flakiness: it counts run-to-run model non-determinism even when the assertion", + " tolerated it (e.g. an `anyof` clarify or a `duplicateOfPrevious` extra action).", + "- Only the LLM **translation** step's tokens are counted; grammar/cache-resolved", + " attempts consume no translation tokens and are deterministic, so they neither", + " add tokens nor cause flaky failures.", + "- When a run has zero events, MTTF is unbounded; the charts and `≥` cells use", + " the run's token total as a lower bound.", + "", + "## Regenerate", + "", + "From `ts/`:", + "", + "```powershell", + '$env:TRANSLATION_RELIABILITY_DIR = "$PWD/tmp/reliability"', + "Remove-Item $env:TRANSLATION_RELIABILITY_DIR -Recurse -Force -ErrorAction SilentlyContinue", + "pnpm --filter default-agent-provider build", + "Push-Location packages/defaultAgentProvider", + "pnpm run jest-esm --testPathPattern=translate --forceExit", + "Pop-Location", + "node packages/defaultAgentProvider/dist/test/reliability/report.js tmp/reliability packages/defaultAgentProvider/test/reliability/README.md ", + "Remove-Item Env:\\TRANSLATION_RELIABILITY_DIR", + "```", + "", + "Each run appends one point to `reliability/history.json` (kept in git) and", + "re-renders the charts above.", + "", + ]; + + fs.mkdirSync(path.dirname(outDoc), { recursive: true }); + fs.writeFileSync(outDoc, lines.join("\n")); + + console.log(`Wrote ${outDoc}`); + console.log( + `suites=${suites.length} attempts=${attempts} failures=${failures} ` + + `variations=${variations} tokens=${tokens}`, + ); + console.log( + `MTTF(failures)=${mttfLabel(tokens, failures)} ` + + `MTTF(variations)=${mttfLabel(tokens, variations)}`, + ); + console.log(`history: ${history.length} run(s) -> ${historyPath}`); +} + +main(); diff --git a/ts/packages/defaultAgentProvider/test/translateTestCommon.ts b/ts/packages/defaultAgentProvider/test/translateTestCommon.ts index 0957cde1e..79bc5aded 100644 --- a/ts/packages/defaultAgentProvider/test/translateTestCommon.ts +++ b/ts/packages/defaultAgentProvider/test/translateTestCommon.ts @@ -52,6 +52,19 @@ function isAnyOfActionMatch(a: ActionMatch): a is AnyOfActionMatch { return typeof a === "object" && "anyof" in a; } +// Sentinel usable only in `extraActions`: the trailing action passes only when +// it is an exact duplicate of the action immediately preceding it. Scopes the +// tolerance tightly to the known flake where the model repeats its final action +// (e.g. a duplicated dispatcher.pendingRequestAction), instead of accepting an +// arbitrary extra action of a given type. +type DuplicateOfPreviousMatch = { duplicateOfPrevious: true }; +type ExtraActionMatch = ActionMatch | DuplicateOfPreviousMatch; +function isDuplicateOfPreviousMatch( + a: ExtraActionMatch, +): a is DuplicateOfPreviousMatch { + return typeof a === "object" && a !== null && "duplicateOfPrevious" in a; +} + function toActionMatchWithAlternates( match: OneActionMatch, ): ActionMatchWithAlternates { @@ -97,8 +110,11 @@ export type TranslateTestStep = { // 0..extraActions.length additional actions, each validated in order against // the corresponding entry here. Use for multi-action requests with variable // tails (e.g. an extra pendingRequestAction that defers "add the filtered - // tracks to the playlist"). - extraActions?: ActionMatch | ActionMatch[]; + // tracks to the playlist"). A `{ duplicateOfPrevious: true }` entry passes + // only when the trailing action is an exact duplicate of the action right + // before it - tolerating the model repeating its final action without + // accepting an arbitrary extra one. + extraActions?: ExtraActionMatch | ExtraActionMatch[]; }; export type TranslateTestEntry = TranslateTestStep | TranslateTestStep[]; @@ -162,6 +178,41 @@ function isTransientRequestError(error: string | undefined): boolean { ); } +// Suite-wide reliability accounting helper (opt-in via TRANSLATION_RELIABILITY_DIR). +// Writes one JSON tally per suite that reliability/report.ts aggregates into the +// mean-translation-tokens-between-flaky-failures doc. A "variation" is an attempt +// whose action signature differs from the most common (modal) signature for the +// same request - run-to-run model non-determinism, whether or not the assertion +// tolerated it. +function writeReliabilityTally( + dir: string, + name: string, + attempts: number, + failures: number, + outcomes: Map, + tokens: number, +) { + let variations = 0; + for (const signatures of outcomes.values()) { + const counts = new Map(); + for (const signature of signatures) { + counts.set(signature, (counts.get(signature) ?? 0) + 1); + } + const modal = Math.max(...counts.values()); + variations += signatures.length - modal; + } + fs.mkdirSync(dir, { recursive: true }); + const safeName = name.replace(/[^a-z0-9]+/gi, "_"); + fs.writeFileSync( + path.join(dir, `${safeName}.json`), + JSON.stringify( + { name, attempts, failures, variations, tokens }, + null, + 2, + ), + ); +} + export async function defineTranslateTest( name: string, dataFiles: string[], @@ -189,6 +240,18 @@ export async function defineTranslateTest( i, ] as const, ); + + // Opt-in suite-wide reliability accounting. When TRANSLATION_RELIABILITY_DIR + // is set, tally attempts / test failures / translation tokens (and per-request + // outcome variations) for this suite and write a JSON at afterAll; + // reliability/report.ts aggregates all suites into the reliability doc + // (mean translation tokens between flaky failures across the whole suite). + const reliabilityDir = process.env.TRANSLATION_RELIABILITY_DIR; + let relAttempts = 0; + let relFailures = 0; + let relTokens = 0; + const relOutcomes = new Map(); + describe(`${name} action stability`, () => { let dispatchers: Dispatcher[] = []; let dispatcherP: Promise | undefined; @@ -290,7 +353,31 @@ export async function defineTranslateTest( } try { const result = await runOneStep(step, dispatcher); - validateCommandResult(step, result); + if (reliabilityDir !== undefined) { + relAttempts++; + relTokens += + result?.tokenUsage?.total_tokens ?? 0; + const signature = (result?.actions ?? []) + .map( + (a) => + `${a.schemaName}.${a.actionName}`, + ) + .join("+"); + const seen = relOutcomes.get(step.request); + if (seen === undefined) { + relOutcomes.set(step.request, [signature]); + } else { + seen.push(signature); + } + } + try { + validateCommandResult(step, result); + } catch (e) { + if (reliabilityDir !== undefined) { + relFailures++; + } + throw e; + } } finally { if (step.skipGrammar) { await awaitCommand( @@ -309,6 +396,16 @@ export async function defineTranslateTest( ); }); afterAll(async () => { + if (reliabilityDir !== undefined) { + writeReliabilityTally( + reliabilityDir, + name, + relAttempts, + relFailures, + relOutcomes, + relTokens, + ); + } const p = dispatchers.map((d) => d.close()); await Promise.allSettled(p); dispatchers = []; @@ -388,23 +485,55 @@ function validateCommandResult( // `expected` is required and validated as the leading prefix. // `extraActions` are optional trailing actions: the result may contain // 0..extraActions.length of them, each validated in order against the - // corresponding entry. - const extraMatches = - step.extraActions !== undefined - ? normalizeActionMatches(step.extraActions) - : []; + // corresponding entry. A `{ duplicateOfPrevious: true }` entry passes + // only when the trailing action is an exact duplicate of the action + // immediately before it. + const extraSpecs: ExtraActionMatch[] = + step.extraActions === undefined + ? [] + : Array.isArray(step.extraActions) + ? step.extraActions + : [step.extraActions]; expect(actions.length).toBeGreaterThanOrEqual(actionMatches.length); expect(actions.length).toBeLessThanOrEqual( - actionMatches.length + extraMatches.length, + actionMatches.length + extraSpecs.length, ); - const allMatches = [...actionMatches, ...extraMatches]; for (let i = 0; i < actions.length; i++) { - validateExpectedAction(allMatches[i], actions[i]); + if (i < actionMatches.length) { + validateExpectedAction(actionMatches[i], actions[i]); + continue; + } + const extra = extraSpecs[i - actionMatches.length]; + if (isDuplicateOfPreviousMatch(extra)) { + checkDuplicateOfPreviousAction(actions, i); + } else { + validateExpectedAction( + normalizeActionMatches(extra)[0], + actions[i], + ); + } } } } +// A `{ duplicateOfPrevious: true }` extraActions slot passes only when the +// trailing action is an exact duplicate of the action immediately preceding it. +// Both actions are normalized (and their run-to-run `entities` metadata dropped, +// as in checkPossibleMatch) before the equality check. +function checkDuplicateOfPreviousAction(actions: TypeAgentAction[], i: number) { + const normalizeForCompare = (a: TypeAgentAction) => { + const n = structuredClone(a); + normalizeAction(n); + normalizeUrlParams(n); + delete n.entities; + return n; + }; + expect(normalizeForCompare(actions[i])).toEqual( + normalizeForCompare(actions[i - 1]), + ); +} + type PossibleMatch = { action: TypeAgentAction; partial: boolean; @@ -474,15 +603,13 @@ function checkPossibleMatch( action: TypeAgentAction, possibleMatch: PossibleMatch, ) { - // action.entities is resolved-reference metadata the translator may or may - // not attach run-to-run (e.g. a "grocery" list entity whose items facet - // reflects the current, history-dependent list contents). It is not part of - // the translated action shape most cases care about, so only assert it when - // the expected action explicitly specifies entities (e.g. - // translate-image-history-e2e.json). Otherwise drop it so its - // non-determinism can't cause spurious toEqual mismatches. + // Drop action.entities before comparing. It is resolved-reference metadata + // that entity resolution may or may not attach run-to-run (e.g. a + // history-dependent "grocery" list entity), not part of the translated + // action shape, and no expected action asserts it. Ignoring it avoids + // spurious toEqual mismatches. let actual = action; - if (possibleMatch.action.entities === undefined && action.entities) { + if (action.entities) { actual = { ...action }; delete actual.entities; } diff --git a/ts/packages/dispatcher/dispatcher/README.AUTOGEN.md b/ts/packages/dispatcher/dispatcher/README.AUTOGEN.md index 539382c6d..b2e83eead 100644 --- a/ts/packages/dispatcher/dispatcher/README.AUTOGEN.md +++ b/ts/packages/dispatcher/dispatcher/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # agent-dispatcher — AI-generated documentation @@ -12,7 +12,7 @@ ## Overview -The TypeAgent Dispatcher is a TypeScript library that serves as the core component of the TypeAgent ecosystem. It processes user inputs, translates natural language into structured actions using large language models (LLMs), and coordinates interactions across various application agents. The Dispatcher is designed to integrate with multiple front ends, such as the TypeAgent Shell and CLI, and supports an extensible architecture for building personal agents with natural language interfaces. +The TypeAgent Dispatcher is a TypeScript library that acts as the central component of the TypeAgent ecosystem. It processes user inputs, translates natural language into structured actions using large language models (LLMs), and coordinates interactions across various application agents. The Dispatcher is designed to integrate with multiple front ends, such as the TypeAgent Shell and CLI, and supports an extensible architecture for building personal agents with natural language interfaces. ## What it does @@ -236,6 +236,6 @@ _9 environment variables referenced from `./src/` (set in `ts/.env` or your shel --- -_Auto-generated against commit `2353118509b02d4e8cf94303aea43ad6726f610c` on `2026-07-21T19:50:04.245Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter agent-dispatcher docs:verify-links` to spot-check._ +_Auto-generated against commit `8192acc149ab6f52296dcc71202b20253b6a3089` on `2026-07-22T05:16:33.340Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter agent-dispatcher docs:verify-links` to spot-check._ diff --git a/ts/packages/dispatcher/dispatcher/src/context/chatHistory.ts b/ts/packages/dispatcher/dispatcher/src/context/chatHistory.ts index 84c13187a..d6f18e148 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/chatHistory.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/chatHistory.ts @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { ActionResultActivityContext, Entity } from "@typeagent/agent-sdk"; +import { + ActionResultActivityContext, + AppAction, + Entity, +} from "@typeagent/agent-sdk"; import { CachedImageWithDetails, extractRelevantExifTags, @@ -24,6 +28,10 @@ type AssistantEntry = { entities?: Entity[] | undefined; additionalInstructions?: string[] | undefined; activityContext?: ActionResultActivityContext | undefined; + // The action that was executed to produce this result, if any. Rendered + // into the translation prompt (behind promptConfig.recentActions) so the + // model can see that the request was already carried out. + action?: AppAction | undefined; }; type ChatHistoryEntry = UserEntry | AssistantEntry; @@ -34,6 +42,7 @@ export type ChatHistoryInputAssistant = { entities?: Entity[]; additionalInstructions?: string[]; activityContext?: ActionResultActivityContext; + action?: AppAction; }; export type ChatHistoryInputEntry = { @@ -52,6 +61,7 @@ function convertAssistantMessage( entities: message.entities, additionalInstructions: message.additionalInstructions, activityContext: message.activityContext, + action: message.action, }); } @@ -94,8 +104,14 @@ export interface ChatHistory { entities?: Entity[], additionalInstructions?: string[], activityContext?: ActionResultActivityContext, + action?: AppAction, ): void; getCurrentInstructions(): string[] | undefined; + // The action(s) executed for the assistant turns within the recent chat + // history window (the same turns shown via getPromptSections), oldest + // first, capped to the most recent `limit`. Lets the model see which + // requests were already carried out so it doesn't re-issue them. + getRecentActions(limit: number): AppAction[] | undefined; getPromptSections(): PromptSection[]; getLastActivityContextInfo(): | { @@ -134,29 +150,31 @@ export interface ChatHistory { export function createChatHistory(init: boolean): ChatHistory { let enabled = init; const entries: ChatHistoryEntry[] = []; + // Index of the oldest entry whose trailing run fits within maxChars of + // text. Shared by getPromptSections and getRecentActions so the executed- + // action list covers exactly the turns shown as recent chat history. + const recentStartIndex = (maxChars: number): number => { + let totalLength = 0; + let i = entries.length - 1; + while (i >= 0) { + const nextLength = entries[i].text.length; + if (nextLength + totalLength > maxChars) { + ++i; + break; + } + totalLength += nextLength; + --i; + } + return i < 0 ? 0 : i; + }; return { enable(value: boolean) { enabled = value; }, getPromptSections(maxChars = 2000) { const sections: PromptSection[] = []; - // Find the last N that can fit the character quota - let totalLength = 0; - let i: number = entries.length - 1; - // Get the range of sections that could be pushed on, NEWEST first - while (i >= 0) { - const nextLength = entries[i].text.length; - if (nextLength + totalLength > maxChars) { - ++i; - break; - } - totalLength += nextLength; - --i; - } - if (i < 0) { - i = 0; - } - for (; i < entries.length; ++i) { + // Find the last N entries that fit the character quota. + for (let i = recentStartIndex(maxChars); i < entries.length; ++i) { const entry = entries[i]; if (entry.text.length > 0) { @@ -225,6 +243,31 @@ export function createChatHistory(init: boolean): ChatHistory { } return instructions.length > 0 ? instructions : undefined; }, + getRecentActions( + limit: number, + maxChars = 2000, + ): AppAction[] | undefined { + // Executed action(s) for the assistant turns within the same recent + // window shown as chat history, oldest first, so the model can see + // which requests were already carried out and not re-issue them. + // Capped to the most recent `limit` actions. + if (limit <= 0) { + return undefined; + } + const actions: AppAction[] = []; + for (let i = recentStartIndex(maxChars); i < entries.length; ++i) { + const entry = entries[i]; + if (entry.role === "assistant" && entry.action) { + actions.push(entry.action); + } + } + if (actions.length === 0) { + return undefined; + } + return actions.length > limit + ? actions.slice(actions.length - limit) + : actions; + }, getLastActivityContextInfo() { if (entries.length === 0) { return undefined; @@ -256,6 +299,7 @@ export function createChatHistory(init: boolean): ChatHistory { entities?: Entity[], additionalInstructions?: string[], activityContext?: ActionResultActivityContext, + action?: AppAction, ): void { if (enabled) { entries.push({ @@ -265,6 +309,7 @@ export function createChatHistory(init: boolean): ChatHistory { entities: structuredClone(entities), // make a copy so that it doesn't get modified by others later. additionalInstructions, activityContext: structuredClone(activityContext), + action: structuredClone(action), }); } }, @@ -460,6 +505,9 @@ export function createChatHistory(init: boolean): ChatHistory { if (entry.activityContext) { assistantEntry.activityContext = entry.activityContext; } + if (entry.action) { + assistantEntry.action = structuredClone(entry.action); + } (currInput.assistant as any).push(assistantEntry); } } @@ -508,6 +556,13 @@ const assistantInputSchema = sc.obj({ activityEndAction: sc.optional(sc.any()), }), ), + action: sc.optional( + sc.obj({ + actionName: sc.string(), + schemaName: sc.optional(sc.string()), + parameters: sc.optional(sc.any()), + }), + ), }); const messageInputSchema = sc.obj({ diff --git a/ts/packages/dispatcher/dispatcher/src/context/chatHistoryPrompt.ts b/ts/packages/dispatcher/dispatcher/src/context/chatHistoryPrompt.ts index 101ad0f56..0477145f4 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/chatHistoryPrompt.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/chatHistoryPrompt.ts @@ -160,6 +160,15 @@ export function createTypeAgentRequestPrompt( prompts.push(...additionalInstructions); } + const recentActions = history?.actions; + if (recentActions !== undefined && recentActions.length > 0) { + prompts.push("###"); + prompts.push( + "The following action(s) were already executed in the recent chat history, oldest first. Do NOT re-issue them unless the current user request explicitly asks to repeat them:", + ); + prompts.push(JSON.stringify(recentActions, undefined, 2)); + } + prompts.push("###"); prompts.push("The latest assistant response:"); prompts.push( diff --git a/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/clarifyActionSchema.ts b/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/clarifyActionSchema.ts index df144c64d..07a78059c 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/clarifyActionSchema.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/clarifyActionSchema.ts @@ -20,8 +20,12 @@ export interface ClarifyMultiplePossibleActionName { }; } -// The user request is for an action, but some of the parameter is not specified or cannot be infer from the context. -// Ask the user for clarification for the missing parameter of an known action. Don't clarify unknown action. +// The user request is for a known action, but a required parameter is missing +// entirely: the user gave no value for it, and no pronoun, reference, or memory +// points at one. Ask the user to supply the missing parameter. Don't clarify +// "unknown" action. If the value IS referred to but can't be resolved (e.g. +// "the one we talked about", "it", "that file", "the one from before"), use +// ClarifyUnresolvedReference instead. export interface ClarifyMissingParameter { actionName: "clarifyMissingParameter"; parameters: { @@ -34,7 +38,10 @@ export interface ClarifyMissingParameter { }; } -// The user request is for an action, but parameters are referring to unresolved pronouns or references that is not found in the request or recent chat context. +// The user request is for a known action and does not refer to the parameter's +// value directly, but rather through a pronoun, reference, or memory that can't be resolved +// directly from the request or chat history (e.g. "from the movie we talked about +// yesterday", "print that file", "it"). Ask the user what the reference points to. export interface ClarifyUnresolvedReference { actionName: "clarifyUnresolvedReference"; parameters: { @@ -42,7 +49,7 @@ export interface ClarifyUnresolvedReference { request: string; actionName: string; parameterName: string; - reference: string; // words of the unresolved pronoun or reference + reference: string; // words of the unresolved pronoun, reference, or memory clarifyingQuestion: string; }; } diff --git a/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/reasoningActionSchema.ts b/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/reasoningActionSchema.ts index 2e7094b48..fd12be762 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/reasoningActionSchema.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/dispatcher/schema/reasoningActionSchema.ts @@ -7,6 +7,11 @@ export interface ReasoningAction { parameters: { // The original user request originalRequest: string; + // Why this request needs multi-step reasoning instead of a single known + // action. Name the action(s) or schema that came closest and say what + // made a direct translation insufficient. Be specific: this is used to + // find gaps in the action schemas. + reason?: string; // JSON-serialized action object — agents populate this when redirecting a // translated action so the reasoning loop knows what was intended. attemptedAction?: string; diff --git a/ts/packages/dispatcher/dispatcher/src/context/memory.ts b/ts/packages/dispatcher/dispatcher/src/context/memory.ts index f9c7acdae..193529c8f 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/memory.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/memory.ts @@ -17,6 +17,7 @@ import type { ActionContext, ActionResult, ActionResultActivityContext, + AppAction, Entity, ParsedCommandParams, } from "@typeagent/agent-sdk"; @@ -134,6 +135,7 @@ export function addResultToMemory( entities?: Entity[], additionalInstructions?: string[], activityContext?: ActionResultActivityContext, + action?: AppAction, ) { context.chatHistory.addAssistantEntry( message, @@ -141,6 +143,7 @@ export function addResultToMemory( entities, additionalInstructions, activityContext, + action, ); if (context.actionResultKnowledgeExtraction) { @@ -216,6 +219,7 @@ export function addActionResultToMemory( combinedEntities, result.additionalInstructions, result.activityContext, + executableAction.action, ); } } diff --git a/ts/packages/dispatcher/dispatcher/src/context/session.ts b/ts/packages/dispatcher/dispatcher/src/context/session.ts index 1eaa9bc15..90cae54cb 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/session.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/session.ts @@ -97,6 +97,13 @@ export type DispatcherConfig = { stream: boolean; promptConfig: { additionalInstructions: boolean; + // Include the structured action(s) recently executed in the chat + // history in the translation prompt, so the model can see which + // requests were already carried out and avoid re-issuing them. + recentActions: boolean; + // Max number of recently executed actions to include (most recent + // ones), 0-100 (0 disables the feature). + recentActionsLimit: number; }; switch: { fixed: string; // fixed first schema to use, ignore embedding if set @@ -381,10 +388,16 @@ const defaultSessionConfig: SessionConfig = { request: DispatcherName, translation: { enabled: true, - model: "", + // Default translation model. "GPT_4_1" resolves to the gpt-4.1 + // deployment in both config.local.yaml and the CI build-pipeline-kv + // config, keeping local dev and CI on the same model. Override per + // session with `@config translation model `. + model: "GPT_4_1", stream: true, promptConfig: { additionalInstructions: true, + recentActions: true, + recentActionsLimit: 3, }, switch: { fixed: "", diff --git a/ts/packages/dispatcher/dispatcher/src/context/system/handlers/configCommandHandlers.ts b/ts/packages/dispatcher/dispatcher/src/context/system/handlers/configCommandHandlers.ts index d0e02ed11..01c9ad65a 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/system/handlers/configCommandHandlers.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/system/handlers/configCommandHandlers.ts @@ -1100,6 +1100,40 @@ class HistoryLimitCommandHandler implements CommandHandler { } } +class RecentActionsLimitCommandHandler implements CommandHandler { + public readonly description = + "Set the max number of recently executed actions included in translation history (0-100, 0 = off)"; + public readonly parameters = { + args: { + limit: { + description: "Number of actions (0-100, 0 = off)", + type: "number", + }, + }, + } as const; + public async run( + context: ActionContext, + params: ParsedCommandParams, + ) { + const limit = params.args.limit; + if (limit < 0 || limit > 100) { + throw new Error("Limit must be an integer between 0 and 100"); + } + await changeContextConfig( + { + translation: { + promptConfig: { recentActionsLimit: limit }, + }, + }, + context, + ); + displayResult( + `Recently executed actions included in translation is set to ${limit}`, + context, + ); + } +} + class GrammarSystemCommandHandler implements CommandHandler { public readonly description = "Set grammar system (completionBased or nfa)"; public readonly parameters = { @@ -1316,6 +1350,27 @@ const configTranslationCommandHandlers: CommandHandlerTable = { }, }, + recentActions: { + description: + "Include recently executed actions in translation history", + commands: { + ...getToggleCommandHandlers( + "recently executed actions in translation", + async (context, enable: boolean) => { + await changeContextConfig( + { + translation: { + promptConfig: { recentActions: enable }, + }, + }, + context, + ); + }, + ), + limit: new RecentActionsLimitCommandHandler(), + }, + }, + stream: getToggleHandlerTable( "streaming translation", async (context, enable: boolean) => { diff --git a/ts/packages/dispatcher/dispatcher/src/context/system/schema/grammarActionSchema.ts b/ts/packages/dispatcher/dispatcher/src/context/system/schema/grammarActionSchema.ts index 429433455..bd88a940a 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/system/schema/grammarActionSchema.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/system/schema/grammarActionSchema.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +// Lists the active grammar rules for the supplied agent. export type ListRulesAction = { actionName: "listRules"; parameters?: { diff --git a/ts/packages/dispatcher/dispatcher/src/translation/interpretRequest.ts b/ts/packages/dispatcher/dispatcher/src/translation/interpretRequest.ts index 3d995827a..cbdb51e39 100644 --- a/ts/packages/dispatcher/dispatcher/src/translation/interpretRequest.ts +++ b/ts/packages/dispatcher/dispatcher/src/translation/interpretRequest.ts @@ -53,10 +53,16 @@ export function createHistoryContext( .additionalInstructions ? context.chatHistory.getCurrentInstructions() : undefined; + const actions = translateConfig.promptConfig.recentActions + ? context.chatHistory.getRecentActions( + translateConfig.promptConfig.recentActionsLimit, + ) + : undefined; return { promptSections, entities, additionalInstructions, + actions, activityContext: context.activityContext, }; }