From ee89ad92959d9df3ffeb7b357f65bda0e82d0647 Mon Sep 17 00:00:00 2001 From: MK Date: Mon, 24 Aug 2026 23:19:55 -0400 Subject: [PATCH 1/2] otel: adopt gen_ai.* semantic conventions on tool + agent spans Forge emitted a partial GenAI attribute set and published tool telemetry under proprietary forge.tool.* keys that GenAI-native backends don't recognize. This adopts the OTel GenAI semantic conventions. Tool spans (tool.) now emit, replacing forge.tool.* (which never shipped to production, so no dual-emit window): - gen_ai.operation.name=execute_tool, gen_ai.tool.name, gen_ai.tool.call.id, gen_ai.tool.type (function | extension for MCP-backed tools) - gen_ai.tool.call.arguments / .result / .description (opt-in, CaptureContent) - error.type on failure (replaces forge.tool.error) - MCP tools (namespaced "__") additionally carry mcp.method.name=tools/call. mcp.session.id / mcp.protocol.version need the MCP manager plumbed to the executor and are deferred. agent.execute now emits gen_ai.provider.name, gen_ai.agent.id/.name/.version (from forge.yaml), gen_ai.conversation.id (Forge session / A2A task id), and gen_ai.tool.definitions (opt-in). llm.completion now emits gen_ai.operation.name=chat, gen_ai.provider.name, gen_ai.response.id, and gen_ai.response.model (wired via a new ChatResponse.Model populated from the Anthropic/OpenAI response bodies; falls back to the request model). gen_ai.system is kept one release as a deprecated alias of gen_ai.provider.name. attrs.go constants updated in one sweep. Tests assert the new keys on tool/agent/llm spans (incl. the MCP extension path); docs/core-concepts/observability-tracing.md updated. Refs initializ/forge#421 --- docs/core-concepts/observability-tracing.md | 35 ++-- forge-cli/runtime/runner.go | 2 + forge-core/llm/providers/anthropic.go | 2 + forge-core/llm/providers/openai.go | 4 +- forge-core/llm/types.go | 7 + forge-core/observability/attrs.go | 110 +++++++++++-- forge-core/runtime/loop.go | 113 +++++++++++-- forge-core/runtime/loop_spans_content_test.go | 30 ++-- forge-core/runtime/loop_spans_test.go | 150 +++++++++++++++++- 9 files changed, 403 insertions(+), 50 deletions(-) diff --git a/docs/core-concepts/observability-tracing.md b/docs/core-concepts/observability-tracing.md index 27dc1b25..2124ca0a 100644 --- a/docs/core-concepts/observability-tracing.md +++ b/docs/core-concepts/observability-tracing.md @@ -212,14 +212,25 @@ Forge mixes OTel GenAI semconv with Forge-specific `forge.*` namespaced attribut | `forge.correlation_id` | `agent.execute` | inbound `X-Forge-Correlation-Id` | | `forge.loop.iteration` | `agent.execute` (set at End) | turn count | | `forge.task.final_state` | `agent.execute` (set at End) | `completed` / `failed` / `canceled` | -| `gen_ai.system` | `agent.execute`, `llm.completion` | `"anthropic"`, `"openai"`, `"ollama"` | -| `gen_ai.request.model` / `.response.model` | `llm.completion` | provider request/response model | +| `gen_ai.provider.name` | `agent.execute`, `llm.completion` | `"anthropic"`, `"openai"`, `"ollama"` — current key (see deprecation note below) | +| `gen_ai.system` | `agent.execute`, `llm.completion` | same value as `gen_ai.provider.name`; **deprecated**, emitted one release for compatibility | +| `gen_ai.operation.name` | `llm.completion` (`chat`), `tool.` (`execute_tool`) | operation kind | +| `gen_ai.agent.id` / `.name` / `.version` | `agent.execute` | `forge.yaml` `agent_id` (id + name) / `version` | +| `gen_ai.conversation.id` | `agent.execute` | Forge session id (A2A task id) | +| `gen_ai.request.model` | `agent.execute`, `llm.completion` | requested model | +| `gen_ai.response.model` | `llm.completion` | vendor-reported model (falls back to request model) | +| `gen_ai.response.id` | `llm.completion` | provider completion id | | `gen_ai.usage.input_tokens` / `.output_tokens` | `llm.completion` | provider usage block | | `gen_ai.response.finish_reasons` | `llm.completion` | provider stop reason | -| `forge.tool.name` | `tool.` | tool function name | -| `forge.tool.error` | `tool.` | error message on failure | +| `gen_ai.tool.name` | `tool.` | tool function name | +| `gen_ai.tool.call.id` | `tool.` | LLM-assigned tool-call id | +| `gen_ai.tool.type` | `tool.` | `function` (builtin/skill) or `extension` (MCP-backed) | +| `mcp.method.name` | `tool.` (MCP only) | `tools/call` | +| `error.type` | `tool.` (on failure) | `tool_execution_error` | -Tool errors do **not** fail the outer `agent.execute` span — they surface to the LLM as text and the loop continues. The tool span carries the failure detail so operators can pivot from a trace to the specific failed invocation. +Tool spans follow the OTel **GenAI semantic conventions** (`gen_ai.tool.*`) — these replaced the former proprietary `forge.tool.*` keys, which never shipped to production. MCP-backed tools (namespaced `__`) are typed `extension` and additionally carry `mcp.method.name=tools/call`. (`mcp.session.id` / `mcp.protocol.version` require plumbing the MCP manager to the executor and are tracked as a follow-up.) + +Tool errors do **not** fail the outer `agent.execute` span — they surface to the LLM as text and the loop continues. The tool span carries the failure detail (`error.type` + span status `Error`) so operators can pivot from a trace to the specific failed invocation. ### Span content capture @@ -227,18 +238,20 @@ Prompts, completions, tool args, and tool results are **off by default** — Pha | `forge.yaml` knob | Span | Attribute keys added when `capture_content: true` | |---|---|---| -| (always) | `llm.completion` | `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.response.finish_reasons` | +| (always) | `agent.execute` | `gen_ai.provider.name`, `gen_ai.agent.id`, `gen_ai.agent.name`, `gen_ai.agent.version`, `gen_ai.conversation.id`, `gen_ai.request.model` | +| `capture_content: true` | `agent.execute` | `gen_ai.tool.definitions` (JSON array of the tool catalog available to the agent — potentially large, hence opt-in) | +| (always) | `llm.completion` | `gen_ai.operation.name` (`chat`), `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`, `gen_ai.response.id`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.response.finish_reasons` | | `capture_content: true` | `llm.completion` | `gen_ai.input.messages` (JSON array of role+content pairs sent to the model), `gen_ai.output.messages` (JSON single-element array of role+content for the model's response) — current OTel GenAI semconv, supersedes the deprecated flat-string `gen_ai.prompt` / `gen_ai.completion` | -| (always) | `tool.` | `forge.tool.name`, `forge.tool.error` (on failure) | -| `capture_content: true` | `tool.` | `forge.tool.args` (raw arguments JSON), `forge.tool.result` (raw output) | +| (always) | `tool.` | `gen_ai.operation.name` (`execute_tool`), `gen_ai.tool.name`, `gen_ai.tool.call.id`, `gen_ai.tool.type`, `mcp.method.name` (MCP only), `error.type` (on failure) | +| `capture_content: true` | `tool.` | `gen_ai.tool.call.arguments` (raw arguments JSON), `gen_ai.tool.call.result` (raw output), `gen_ai.tool.description` (from the tool definition) | When `capture_content: true` and `redact: true` (the default when capture is on), attribute values pass through a redactor that scrubs the same vendor secret-token shapes the runtime guardrails default rules cover (Anthropic `sk-ant-…`, OpenAI `sk-…`, GitHub `ghp_/gho_/ghs_/github_pat_…`, AWS `AKIA…`, Slack `xoxb-/xoxp-…`, RSA/EC/OPENSSH/PRIVATE key blocks, Telegram bot tokens). Matched values become `[REDACTED]`. Setting `redact: false` is the enterprise raw-capture path — content is stamped verbatim with the byte cap still applied. Every captured value is byte-capped at **4 KiB** (below the 5 KiB attribute soft-cap most backends apply). When the input exceeds the cap, the value ends with a `…[truncated:N]` marker where `N` is the original byte length. The marker is **byte-identical** to what the audit payload-capture path emits for the same input, so an operator grepping `[truncated:` across span attributes and audit rows sees aligned output. -**Default posture** (no opt-in): the `gen_ai.input.messages`, `gen_ai.output.messages`, `forge.tool.args`, `forge.tool.result` keys are **absent** from spans — not set to empty string. Backends that gate dashboards on "is this key present?" can distinguish "metadata-only by default" from "operator opted in but the field happened to be empty." +**Default posture** (no opt-in): the `gen_ai.input.messages`, `gen_ai.output.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, `gen_ai.tool.description`, and `gen_ai.tool.definitions` keys are **absent** from spans — not set to empty string. Backends that gate dashboards on "is this key present?" can distinguish "metadata-only by default" from "operator opted in but the field happened to be empty." -**OTel semconv versioning note**: the GenAI semantic conventions moved from flat-string (`gen_ai.prompt`, `gen_ai.completion`) to structured (`gen_ai.input.messages`, `gen_ai.output.messages`) attributes. Forge emits only the **current** structured keys. Backends that only recognize the deprecated flat-string attributes will not show prompt / completion text on Forge spans — upgrade the backend's semconv mapping or use a span processor to translate. +**OTel semconv versioning note**: the GenAI semantic conventions moved from flat-string (`gen_ai.prompt`, `gen_ai.completion`) to structured (`gen_ai.input.messages`, `gen_ai.output.messages`) attributes, and renamed `gen_ai.system` → `gen_ai.provider.name`. Forge emits the **current** structured keys and `gen_ai.provider.name`, and continues to emit the deprecated `gen_ai.system` for one release for compatibility. Backends that only recognize the older attributes should upgrade their semconv mapping or use a span processor to translate. ### Guardrail spans (issue #161) @@ -268,7 +281,7 @@ Attribute reference: **Default posture**: `forge.guardrail.evidence` is absent unless `capture_content: true`. The other five attributes are always present when a gate fires (cheap, no PII risk). When tracing is disabled, the noop tracer short-circuits and the spans are not produced at all. -**Content-capture parity**: the evidence attribute uses the exact same `PrepareSpanContent(redact, maxBytes)` pipeline as `gen_ai.input.messages` and `forge.tool.args` — same vendor secret-token scrub, same 4 KiB byte cap, same `…[truncated:N]` marker. Operators get one mental model across all four content streams (LLM input / LLM output / tool args / tool result / guardrail evidence). +**Content-capture parity**: the evidence attribute uses the exact same `PrepareSpanContent(redact, maxBytes)` pipeline as `gen_ai.input.messages` and `gen_ai.tool.call.arguments` — same vendor secret-token scrub, same 4 KiB byte cap, same `…[truncated:N]` marker. Operators get one mental model across all four content streams (LLM input / LLM output / tool args / tool result / guardrail evidence). ## End-to-end propagation (Phase 5) diff --git a/forge-cli/runtime/runner.go b/forge-cli/runtime/runner.go index 2d55484a..c3ffee17 100644 --- a/forge-cli/runtime/runner.go +++ b/forge-cli/runtime/runner.go @@ -1337,6 +1337,8 @@ func (r *Runner) Run(ctx context.Context) error { Logger: r.logger, ModelName: mc.Client.Model, Provider: mc.Provider, + AgentID: r.cfg.Config.AgentID, + AgentVersion: r.cfg.Config.Version, MaxIterations: 100, CharBudget: charBudget, FilesDir: filepath.Join(r.cfg.WorkDir, ".forge", "files"), diff --git a/forge-core/llm/providers/anthropic.go b/forge-core/llm/providers/anthropic.go index ea3f6304..d4fa2a09 100644 --- a/forge-core/llm/providers/anthropic.go +++ b/forge-core/llm/providers/anthropic.go @@ -333,6 +333,7 @@ func (c *AnthropicClient) convertMessage(m llm.ChatMessage) anthropicMessage { // Anthropic-specific response types. type anthropicResponse struct { ID string `json:"id"` + Model string `json:"model"` Content []anthropicContentBlock `json:"content"` StopReason string `json:"stop_reason"` Usage struct { @@ -375,6 +376,7 @@ func (c *AnthropicClient) parseAnthropicResponse(body io.Reader) (*llm.ChatRespo return &llm.ChatResponse{ ID: resp.ID, + Model: resp.Model, Message: msg, Usage: llm.UsageInfo{ InputTokens: resp.Usage.InputTokens, diff --git a/forge-core/llm/providers/openai.go b/forge-core/llm/providers/openai.go index 92f743c9..bba1bb28 100644 --- a/forge-core/llm/providers/openai.go +++ b/forge-core/llm/providers/openai.go @@ -257,6 +257,7 @@ func derivePromptCacheKey(model string, req *llm.ChatRequest) string { // openaiResponse is the OpenAI-specific response format. type openaiResponse struct { ID string `json:"id"` + Model string `json:"model"` Choices []struct { Message struct { Role string `json:"role"` @@ -284,7 +285,8 @@ func (c *OpenAIClient) parseOpenAIResponse(body io.Reader) (*llm.ChatResponse, e choice := resp.Choices[0] return &llm.ChatResponse{ - ID: resp.ID, + ID: resp.ID, + Model: resp.Model, Message: llm.ChatMessage{ Role: choice.Message.Role, Content: choice.Message.Content, diff --git a/forge-core/llm/types.go b/forge-core/llm/types.go index ebb9a6bb..04fb5182 100644 --- a/forge-core/llm/types.go +++ b/forge-core/llm/types.go @@ -64,6 +64,13 @@ type ChatResponse struct { Message ChatMessage `json:"message"` Usage UsageInfo `json:"usage"` FinishReason string `json:"finish_reason"` + // Model is the model the provider REPORTED generating the response, + // parsed from the provider response body when present (Anthropic / + // OpenAI both echo "model"). Often identical to the request model but + // enterprise routers can substitute a versioned suffix. Surfaced as + // the gen_ai.response.model span attribute; empty when the provider + // does not report it (the executor falls back to the request model). + Model string `json:"model,omitempty"` // Endpoint is the URL the client POSTed to (base URL + provider path). // Set by the provider client so the llm_call audit event can record the // invoked path even when payload capture is off. Internal only (json:"-"). diff --git a/forge-core/observability/attrs.go b/forge-core/observability/attrs.go index e8602015..de952156 100644 --- a/forge-core/observability/attrs.go +++ b/forge-core/observability/attrs.go @@ -24,9 +24,16 @@ package observability // "llm.completion") and reads naturally inline. const ( - // ─── GenAI semconv (draft, pinned to OTel semconv 1.26.0 GenAI). ── + // ─── GenAI semconv (draft). ────────────────────────────────────── // Backends like Honeycomb / Datadog / Grafana Tempo group LLM // activity by these. Naming follows the OTel GenAI spec exactly. + // The core token/model keys below track the 1.26.0 snapshot; the + // agent / operation / tool keys added later (provider.name, + // operation.name, agent.*, conversation.id, tool.*) track the newer + // GenAI registry. Because these are hand-declared string constants + // (not imported from the semconv package), a spec bump is this + // one-file sweep — the resource-level semconv import in otel.go is + // versioned separately and only carries service.* + schema URL. // AttrGenAISystem identifies the LLM vendor: "anthropic", // "openai", "ollama", "openai-compatible". @@ -51,6 +58,70 @@ const ( // "max_tokens", "end_turn", etc. AttrGenAIResponseFinishReasons = "gen_ai.response.finish_reasons" + // AttrGenAIProviderName is the current OTel key for the GenAI vendor + // ("anthropic", "openai", "ollama", ...). It supersedes the + // deprecated AttrGenAISystem (`gen_ai.system`); Forge emits BOTH for + // one release so dashboards keyed on either light up, then drops the + // alias. Same value as AttrGenAISystem. + AttrGenAIProviderName = "gen_ai.provider.name" + + // AttrGenAIOperationName identifies the operation the span measures — + // "chat" on the llm.completion span, "execute_tool" on a tool. + // span. Backends group GenAI activity by (operation.name, provider.name). + AttrGenAIOperationName = "gen_ai.operation.name" + + // OpChat / OpExecuteTool are the two AttrGenAIOperationName values + // Forge emits today. + OpChat = "chat" + OpExecuteTool = "execute_tool" + + // AttrGenAIResponseID is the provider's completion id (Anthropic + // "msg_…", OpenAI "chatcmpl-…"). Read straight from ChatResponse.ID. + AttrGenAIResponseID = "gen_ai.response.id" + + // AttrGenAIConversationID is the stable conversation/thread id. Forge + // maps this to the A2A task id — the session-store key that persists a + // transcript across turns (.forge/sessions/.json). Per semconv + // this MUST be a real thread identifier, never a synthesized UUID or + // trace id; the Forge session id qualifies. + AttrGenAIConversationID = "gen_ai.conversation.id" + + // AttrGenAIAgent* stamp the agent's identity on the agent.execute + // span from forge.yaml. `agent_id` doubles as the human-readable name + // (forge.yaml has no separate name field, matching the A2A card's + // AgentID fallback), so id and name carry the same value today. + AttrGenAIAgentID = "gen_ai.agent.id" + AttrGenAIAgentName = "gen_ai.agent.name" + AttrGenAIAgentVersion = "gen_ai.agent.version" + + // AttrGenAITool* name the tool-call instrumentation on tool. + // spans, following the OTel GenAI tool conventions. These REPLACE the + // former proprietary `forge.tool.*` keys (which never shipped to + // production). Content-bearing tool attributes (arguments / result / + // description / definitions) live in the content-capture block below. + AttrGenAIToolName = "gen_ai.tool.name" + AttrGenAIToolCallID = "gen_ai.tool.call.id" + AttrGenAIToolType = "gen_ai.tool.type" + + // ToolTypeFunction / ToolTypeExtension are the AttrGenAIToolType + // values Forge emits: builtin + skill tools are "function"; MCP-backed + // tools (namespaced "__") are "extension" — an + // agent-side bridge to an external system. + ToolTypeFunction = "function" + ToolTypeExtension = "extension" + + // AttrMCPMethodName is the MCP JSON-RPC method a span represents. For a + // Forge tool call backed by an MCP server it is always "tools/call". + // (mcp.session.id / mcp.protocol.version require plumbing the MCP + // manager down to the executor and are tracked as a follow-up.) + AttrMCPMethodName = "mcp.method.name" + MCPMethodToolsCall = "tools/call" + + // AttrErrorType is the OTel-standard error classification set on a + // tool. span when execution fails (alongside RecordError + + // Status=Error). Replaces the former `forge.tool.error`. + AttrErrorType = "error.type" + // ─── Forge-specific attributes. ────────────────────────────────── // AttrForgeAgentID is the agent_id from forge.yaml — the operator's @@ -93,10 +164,12 @@ const ( // dashboards can chart "iterations per task." AttrForgeLoopIteration = "forge.loop.iteration" - // AttrForgeToolName / AttrForgeToolError name the tool call - // instrumentation. - AttrForgeToolName = "forge.tool.name" - AttrForgeToolError = "forge.tool.error" + // AttrForgeToolName cross-references which tool a guardrail. + // span evaluated (see forge-cli/runtime/guardrails_tracing.go). The + // tool. execution span itself uses the semconv AttrGenAIToolName + // key instead. (The former forge.tool.error was removed in favor of the + // standard AttrErrorType.) + AttrForgeToolName = "forge.tool.name" // AttrForgeTaskFinalState is the terminal A2A TaskState the loop // resolved to — "completed", "failed", "canceled". Set on the @@ -125,13 +198,26 @@ const ( // `gen_ai.completion` flat-string attribute. AttrGenAIOutputMessages = "gen_ai.output.messages" - // AttrForgeToolArgs is the raw arguments JSON the agent passed to - // a tool. Set on tool. spans. - AttrForgeToolArgs = "forge.tool.args" - - // AttrForgeToolResult is the raw output the tool returned. Set on - // tool. spans. - AttrForgeToolResult = "forge.tool.result" + // AttrGenAIToolCallArguments is the raw arguments JSON the agent passed + // to a tool (semconv `gen_ai.tool.call.arguments`). Set on tool. + // spans. Replaces the former `forge.tool.args`. + AttrGenAIToolCallArguments = "gen_ai.tool.call.arguments" + + // AttrGenAIToolCallResult is the raw output the tool returned (semconv + // `gen_ai.tool.call.result`). Set on tool. spans. Replaces the + // former `forge.tool.result`. + AttrGenAIToolCallResult = "gen_ai.tool.call.result" + + // AttrGenAIToolDescription is the tool's description from its + // definition (semconv `gen_ai.tool.description`). Flagged sensitive by + // semconv, so it rides the same CaptureContent opt-in. + AttrGenAIToolDescription = "gen_ai.tool.description" + + // AttrGenAIToolDefinitions is the JSON array of tool definitions + // available to the agent (semconv `gen_ai.tool.definitions`), stamped + // once on the agent.execute span. Potentially large, so it is opt-in + // behind CaptureContent. + AttrGenAIToolDefinitions = "gen_ai.tool.definitions" // ─── Guardrail span attributes (issue #161) ────────────────────── // diff --git a/forge-core/runtime/loop.go b/forge-core/runtime/loop.go index 7be7a16a..f6337efd 100644 --- a/forge-core/runtime/loop.go +++ b/forge-core/runtime/loop.go @@ -94,6 +94,8 @@ type LLMExecutor struct { logger Logger modelName string // resolved model name for context budget provider string // resolved provider name (anthropic, openai, ollama, custom) + agentID string // forge.yaml agent_id — gen_ai.agent.id / .name + agentVersion string // forge.yaml version — gen_ai.agent.version charBudget int // resolved character budget maxToolResultChars int // computed from char budget filesDir string // directory for file_create output @@ -126,6 +128,8 @@ type LLMExecutorConfig struct { Logger Logger ModelName string // model name for context-aware budgeting Provider string // provider name (anthropic, openai, ollama, custom) — for audit attribution + AgentID string // forge.yaml agent_id — stamped as gen_ai.agent.id / gen_ai.agent.name on the agent.execute span + AgentVersion string // forge.yaml version — stamped as gen_ai.agent.version CharBudget int // explicit char budget override (0 = auto from model) FilesDir string // directory for file_create output (default: $TMPDIR/forge-files) SessionMaxAge time.Duration // max idle time before session recovery is skipped (0 = 30m default) @@ -194,6 +198,8 @@ func NewLLMExecutor(cfg LLMExecutorConfig) *LLMExecutor { logger: logger, modelName: cfg.ModelName, provider: cfg.Provider, + agentID: cfg.AgentID, + agentVersion: cfg.AgentVersion, charBudget: budget, maxToolResultChars: toolLimit, filesDir: cfg.FilesDir, @@ -242,16 +248,37 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess }() if task != nil && task.ID != "" { span.SetAttributes(attribute.String(observability.AttrForgeTaskID, task.ID)) + // gen_ai.conversation.id — the Forge session id (A2A task id) is + // the stable thread key that persists a transcript across turns, + // exactly what semconv wants here. + span.SetAttributes(attribute.String(observability.AttrGenAIConversationID, task.ID)) } if cid := CorrelationIDFromContext(ctx); cid != "" { span.SetAttributes(attribute.String(observability.AttrForgeCorrelationID, cid)) } if e.provider != "" { - span.SetAttributes(attribute.String(observability.AttrGenAISystem, e.provider)) + // gen_ai.system is the deprecated key; gen_ai.provider.name is the + // current one. Emit both for one release. + span.SetAttributes( + attribute.String(observability.AttrGenAISystem, e.provider), + attribute.String(observability.AttrGenAIProviderName, e.provider), + ) } if e.modelName != "" { span.SetAttributes(attribute.String(observability.AttrGenAIRequestModel, e.modelName)) } + // gen_ai.agent.* — identity from forge.yaml. agent_id doubles as the + // human-readable name (no separate name field), so id and name carry the + // same value. + if e.agentID != "" { + span.SetAttributes( + attribute.String(observability.AttrGenAIAgentID, e.agentID), + attribute.String(observability.AttrGenAIAgentName, e.agentID), + ) + } + if e.agentVersion != "" { + span.SetAttributes(attribute.String(observability.AttrGenAIAgentVersion, e.agentVersion)) + } mem := NewMemory(e.systemPrompt, e.charBudget, e.modelName) @@ -343,6 +370,26 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess toolDefs = e.tools.ToolDefinitions() } + // Map tool name → description for the gen_ai.tool.description attribute + // on per-call tool spans below. + toolDescByName := make(map[string]string, len(toolDefs)) + for _, td := range toolDefs { + toolDescByName[td.Function.Name] = td.Function.Description + } + + // gen_ai.tool.definitions — the full tool catalog available to the + // agent, stamped once on the agent.execute span. Opt-in (CaptureContent) + // because it can be large; runs through the same redact-then-truncate + // pipeline as the other content attributes. + if e.tracingCfg.CaptureContent && len(toolDefs) > 0 { + if defs, err := json.Marshal(toolDefs); err == nil { + span.SetAttributes(attribute.String( + observability.AttrGenAIToolDefinitions, + PrepareSpanContent(string(defs), e.tracingCfg.Redact, DefaultSpanContentCapBytes), + )) + } + } + // Track large tool outputs so they can be included as file parts // in the response (the LLM may truncate them due to output token limits). var largeToolOutputs []a2a.Part @@ -424,7 +471,9 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // not part of the LLM call's wall-clock measurement. llmCtx, llmSpan := Tracer().Start(ctx, "llm.completion") llmSpan.SetAttributes( + attribute.String(observability.AttrGenAIOperationName, observability.OpChat), attribute.String(observability.AttrGenAISystem, e.provider), + attribute.String(observability.AttrGenAIProviderName, e.provider), attribute.String(observability.AttrGenAIRequestModel, e.modelName), ) // Phase 3.5 (#130) — stamp the structured input messages on the @@ -478,6 +527,18 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess attribute.Int(observability.AttrGenAIUsageInputTokens, resp.Usage.InputTokens), attribute.Int(observability.AttrGenAIUsageOutputTokens, resp.Usage.OutputTokens), ) + if resp.ID != "" { + llmSpan.SetAttributes(attribute.String(observability.AttrGenAIResponseID, resp.ID)) + } + // gen_ai.response.model — the model the vendor reported back. Falls + // back to the request model when the provider does not echo one. + respModel := resp.Model + if respModel == "" { + respModel = e.modelName + } + if respModel != "" { + llmSpan.SetAttributes(attribute.String(observability.AttrGenAIResponseModel, respModel)) + } if resp.FinishReason != "" { llmSpan.SetAttributes(attribute.StringSlice(observability.AttrGenAIResponseFinishReasons, []string{resp.FinishReason})) } @@ -725,7 +786,29 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // by kind without a query. Phase 3.5 (#130) added optional // args/result content capture under CaptureContent + Redact. toolCtx, toolSpan := Tracer().Start(ctx, "tool."+tc.Function.Name) - toolSpan.SetAttributes(attribute.String(observability.AttrForgeToolName, tc.Function.Name)) + // OTel GenAI tool conventions (gen_ai.tool.*). MCP-backed tools + // are namespaced "__" and map to the "extension" + // tool type (an agent-side bridge to an external system); all + // others are "function". + toolType := observability.ToolTypeFunction + isMCPTool := strings.Contains(tc.Function.Name, "__") + if isMCPTool { + toolType = observability.ToolTypeExtension + } + toolSpan.SetAttributes( + attribute.String(observability.AttrGenAIOperationName, observability.OpExecuteTool), + attribute.String(observability.AttrGenAIToolName, tc.Function.Name), + attribute.String(observability.AttrGenAIToolType, toolType), + ) + if tc.ID != "" { + toolSpan.SetAttributes(attribute.String(observability.AttrGenAIToolCallID, tc.ID)) + } + // MCP tool calls carry the MCP method that the executor issues + // downstream. (mcp.session.id / mcp.protocol.version need the MCP + // manager plumbed to the executor — tracked as a follow-up.) + if isMCPTool { + toolSpan.SetAttributes(attribute.String(observability.AttrMCPMethodName, observability.MCPMethodToolsCall)) + } // With compression enabled (deferToolTruncation), relax // tool-internal output caps so the full result reaches the // compression hook instead of being destructively cut inside @@ -733,18 +816,30 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess if e.deferToolTruncation { toolCtx = tools.WithRelaxedLimits(toolCtx) } - if e.tracingCfg.CaptureContent && tc.Function.Arguments != "" { - toolSpan.SetAttributes(attribute.String( - observability.AttrForgeToolArgs, - PrepareSpanContent(tc.Function.Arguments, e.tracingCfg.Redact, DefaultSpanContentCapBytes), - )) + // gen_ai.tool.description + gen_ai.tool.call.arguments are + // content-bearing; semconv flags them sensitive, so they ride + // the CaptureContent opt-in. + if e.tracingCfg.CaptureContent { + if desc := toolDescByName[tc.Function.Name]; desc != "" { + toolSpan.SetAttributes(attribute.String( + observability.AttrGenAIToolDescription, + PrepareSpanContent(desc, e.tracingCfg.Redact, DefaultSpanContentCapBytes), + )) + } + if tc.Function.Arguments != "" { + toolSpan.SetAttributes(attribute.String( + observability.AttrGenAIToolCallArguments, + PrepareSpanContent(tc.Function.Arguments, e.tracingCfg.Redact, DefaultSpanContentCapBytes), + )) + } } result, execErr := e.tools.Execute(toolCtx, tc.Function.Name, json.RawMessage(tc.Function.Arguments)) toolDuration := time.Since(toolStart) if execErr != nil { toolSpan.RecordError(execErr) toolSpan.SetStatus(codes.Error, execErr.Error()) - toolSpan.SetAttributes(attribute.String(observability.AttrForgeToolError, execErr.Error())) + // OTel-standard error classification (replaces forge.tool.error). + toolSpan.SetAttributes(attribute.String(observability.AttrErrorType, "tool_execution_error")) result = fmt.Sprintf("Error executing tool %s: %s", tc.Function.Name, execErr.Error()) } // Phase 3.5 (#130) — tool result content capture. The @@ -754,7 +849,7 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // will see on the next iteration. if e.tracingCfg.CaptureContent && result != "" { toolSpan.SetAttributes(attribute.String( - observability.AttrForgeToolResult, + observability.AttrGenAIToolCallResult, PrepareSpanContent(result, e.tracingCfg.Redact, DefaultSpanContentCapBytes), )) } diff --git a/forge-core/runtime/loop_spans_content_test.go b/forge-core/runtime/loop_spans_content_test.go index 24cd7abe..4e8ab415 100644 --- a/forge-core/runtime/loop_spans_content_test.go +++ b/forge-core/runtime/loop_spans_content_test.go @@ -284,34 +284,34 @@ func runOneToolCall(t *testing.T, tracingCfg observability.TracingConfig, toolAr // TestExecute_CaptureContentTrue_StampsToolArgsAndResult exercises the // tool-side mirror of the LLM-span content tests. Args from the -// LLM-emitted tool call land at forge.tool.args; tool stdout/return -// lands at forge.tool.result; both go through the same redact + -// truncate pipeline. +// LLM-emitted tool call land at gen_ai.tool.call.arguments; tool +// stdout/return lands at gen_ai.tool.call.result; both go through the +// same redact + truncate pipeline. func TestExecute_CaptureContentTrue_StampsToolArgsAndResult(t *testing.T) { cfg := observability.TracingConfig{CaptureContent: true, Redact: true} span := runOneToolCall(t, cfg, `{"target":"`+awsKeyFixture+`"}`, "deleted "+awsKeyFixture) - args, ok := findAttr(span, observability.AttrForgeToolArgs) + args, ok := findAttr(span, observability.AttrGenAIToolCallArguments) if !ok { - t.Error("forge.tool.args missing when CaptureContent=true") + t.Error("gen_ai.tool.call.arguments missing when CaptureContent=true") } else { if strings.Contains(args, awsKeyFixture) { - t.Errorf("raw AWS key survived redaction on forge.tool.args: %q", args) + t.Errorf("raw AWS key survived redaction on gen_ai.tool.call.arguments: %q", args) } if !strings.Contains(args, RedactionMarker) { - t.Errorf("expected redaction marker in forge.tool.args; got %q", args) + t.Errorf("expected redaction marker in gen_ai.tool.call.arguments; got %q", args) } } - result, ok := findAttr(span, observability.AttrForgeToolResult) + result, ok := findAttr(span, observability.AttrGenAIToolCallResult) if !ok { - t.Error("forge.tool.result missing when CaptureContent=true") + t.Error("gen_ai.tool.call.result missing when CaptureContent=true") } else { if strings.Contains(result, awsKeyFixture) { - t.Errorf("raw AWS key survived redaction on forge.tool.result: %q", result) + t.Errorf("raw AWS key survived redaction on gen_ai.tool.call.result: %q", result) } if !strings.Contains(result, RedactionMarker) { - t.Errorf("expected redaction marker in forge.tool.result; got %q", result) + t.Errorf("expected redaction marker in gen_ai.tool.call.result; got %q", result) } } } @@ -322,10 +322,10 @@ func TestExecute_CaptureContentFalse_NoToolContentAttributes(t *testing.T) { cfg := observability.TracingConfig{CaptureContent: false} span := runOneToolCall(t, cfg, `{"target":"foo"}`, "ok") - if _, ok := findAttr(span, observability.AttrForgeToolArgs); ok { - t.Errorf("CaptureContent=false must not set forge.tool.args") + if _, ok := findAttr(span, observability.AttrGenAIToolCallArguments); ok { + t.Errorf("CaptureContent=false must not set gen_ai.tool.call.arguments") } - if _, ok := findAttr(span, observability.AttrForgeToolResult); ok { - t.Errorf("CaptureContent=false must not set forge.tool.result") + if _, ok := findAttr(span, observability.AttrGenAIToolCallResult); ok { + t.Errorf("CaptureContent=false must not set gen_ai.tool.call.result") } } diff --git a/forge-core/runtime/loop_spans_test.go b/forge-core/runtime/loop_spans_test.go index 1681f53c..3b25be87 100644 --- a/forge-core/runtime/loop_spans_test.go +++ b/forge-core/runtime/loop_spans_test.go @@ -288,12 +288,12 @@ func TestExecuteRecordsToolErrorOnSpan(t *testing.T) { } gotErrAttr := false for _, kv := range toolSpan.Attributes() { - if string(kv.Key) == observability.AttrForgeToolError { + if string(kv.Key) == observability.AttrErrorType { gotErrAttr = true } } if !gotErrAttr { - t.Errorf("tool.broken missing %q attribute", observability.AttrForgeToolError) + t.Errorf("tool.broken missing %q attribute", observability.AttrErrorType) } // Outer span is "completed" — tool errors aren't fatal. @@ -304,3 +304,149 @@ func TestExecuteRecordsToolErrorOnSpan(t *testing.T) { } } } + +// genAIToolRun drives one tool call through the executor and returns the +// recorder so a test can assert span attributes. toolName controls whether +// the tool span is treated as a "function" (builtin/skill) or "extension" +// (MCP-namespaced "__"). +func genAIToolRun(t *testing.T, toolName string) *observability.SpanRecorder { + t.Helper() + tp, rec := observability.NewTestTracerProvider() + SetTracerProvider(tp) + t.Cleanup(func() { + ResetTracerProviderForTest() + _ = tp.Shutdown(context.Background()) + }) + + call := 0 + client := &mockLLMClient{ + chatFunc: func(_ context.Context, _ *llm.ChatRequest) (*llm.ChatResponse, error) { + call++ + if call == 1 { + return &llm.ChatResponse{ + ID: "resp-1", + Model: "claude-sonnet-4-6-20990101", // vendor-reported, differs from request model + Message: llm.ChatMessage{ + Role: llm.RoleAssistant, + ToolCalls: []llm.ToolCall{{ + ID: "tc-42", + Type: "function", + Function: llm.FunctionCall{Name: toolName, Arguments: `{"city":"Tokyo"}`}, + }}, + }, + Usage: llm.UsageInfo{InputTokens: 50, OutputTokens: 10}, + FinishReason: "tool_calls", + }, nil + } + return &llm.ChatResponse{ + ID: "resp-2", + Model: "claude-sonnet-4-6-20990101", + Message: llm.ChatMessage{Role: llm.RoleAssistant, Content: "sunny"}, + Usage: llm.UsageInfo{InputTokens: 60, OutputTokens: 3}, + FinishReason: "stop", + }, nil + }, + } + tools := &mockToolExecutor{ + executeFunc: func(_ context.Context, _ string, _ json.RawMessage) (string, error) { + return "sunny, 20C", nil + }, + } + exec := NewLLMExecutor(LLMExecutorConfig{ + Client: client, + Tools: tools, + MaxIterations: 3, + ModelName: "claude-req", + Provider: "anthropic", + AgentID: "weatherbot", + AgentVersion: "1.2.3", + }) + if _, err := exec.Execute(context.Background(), + &a2a.Task{ID: "task-genai"}, + &a2a.Message{Role: a2a.MessageRoleUser, Parts: []a2a.Part{{Kind: a2a.PartKindText, Text: "hi"}}}); err != nil { + t.Fatalf("Execute: %v", err) + } + return rec +} + +// TestExecuteStampsGenAIAgentAndCompletionAttrs asserts the semconv +// identity/operation attributes on the agent.execute + llm.completion spans. +func TestExecuteStampsGenAIAgentAndCompletionAttrs(t *testing.T) { + rec := genAIToolRun(t, "weather") + + root, ok := rec.FindSpan("agent.execute") + if !ok { + t.Fatal("missing agent.execute span") + } + wantRoot := map[string]string{ + observability.AttrGenAIProviderName: "anthropic", + observability.AttrGenAIAgentID: "weatherbot", + observability.AttrGenAIAgentName: "weatherbot", + observability.AttrGenAIAgentVersion: "1.2.3", + observability.AttrGenAIConversationID: "task-genai", + } + for k, want := range wantRoot { + if got, ok := findAttr(root, k); !ok || got != want { + t.Errorf("agent.execute %s = %q (ok=%v); want %q", k, got, ok, want) + } + } + + llmSpan, ok := rec.FindSpan("llm.completion") + if !ok { + t.Fatal("missing llm.completion span") + } + if got, _ := findAttr(llmSpan, observability.AttrGenAIOperationName); got != observability.OpChat { + t.Errorf("llm.completion operation.name = %q; want %q", got, observability.OpChat) + } + if got, _ := findAttr(llmSpan, observability.AttrGenAIProviderName); got != "anthropic" { + t.Errorf("llm.completion provider.name = %q; want anthropic", got) + } + if got, ok := findAttr(llmSpan, observability.AttrGenAIResponseModel); !ok || got != "claude-sonnet-4-6-20990101" { + t.Errorf("llm.completion response.model = %q (ok=%v); want the vendor-reported model", got, ok) + } + if got, _ := findAttr(llmSpan, observability.AttrGenAIResponseID); got == "" { + t.Error("llm.completion missing gen_ai.response.id") + } +} + +// TestExecuteStampsGenAIToolAttrs asserts the semconv gen_ai.tool.* attrs on +// a function-tool span, and that mcp.method.name is absent for it. +func TestExecuteStampsGenAIToolAttrs(t *testing.T) { + rec := genAIToolRun(t, "weather") + + toolSpan, ok := rec.FindSpan("tool.weather") + if !ok { + t.Fatal("missing tool.weather span") + } + want := map[string]string{ + observability.AttrGenAIOperationName: observability.OpExecuteTool, + observability.AttrGenAIToolName: "weather", + observability.AttrGenAIToolCallID: "tc-42", + observability.AttrGenAIToolType: observability.ToolTypeFunction, + } + for k, wantVal := range want { + if got, ok := findAttr(toolSpan, k); !ok || got != wantVal { + t.Errorf("tool.weather %s = %q (ok=%v); want %q", k, got, ok, wantVal) + } + } + if _, ok := findAttr(toolSpan, observability.AttrMCPMethodName); ok { + t.Error("mcp.method.name must not be set on a non-MCP (function) tool") + } +} + +// TestExecuteMCPToolSpanUsesExtensionType asserts an MCP-namespaced tool +// ("__") is typed "extension" and carries mcp.method.name. +func TestExecuteMCPToolSpanUsesExtensionType(t *testing.T) { + rec := genAIToolRun(t, "linear__create_issue") + + toolSpan, ok := rec.FindSpan("tool.linear__create_issue") + if !ok { + t.Fatal("missing tool.linear__create_issue span") + } + if got, _ := findAttr(toolSpan, observability.AttrGenAIToolType); got != observability.ToolTypeExtension { + t.Errorf("gen_ai.tool.type = %q; want %q for an MCP tool", got, observability.ToolTypeExtension) + } + if got, ok := findAttr(toolSpan, observability.AttrMCPMethodName); !ok || got != observability.MCPMethodToolsCall { + t.Errorf("mcp.method.name = %q (ok=%v); want %q", got, ok, observability.MCPMethodToolsCall) + } +} From 6a1a949ac5c27994fba24ce07f4099460eb698c3 Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 26 Aug 2026 16:20:06 -0400 Subject: [PATCH 2/2] otel: type MCP tools off the MCPSource marker, not the name heuristic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on #422: classifying a tool as `extension` via strings.Contains(name, "__") mistypes NON-MCP namespaced tools — NamespacedSource API per-op tools ("__") share the "__" shape but are not MCP, so they would get gen_ai.tool.type=extension and a spurious mcp.method.name=tools/call. Add Registry.IsMCPTool(name), which type-asserts the registered tool to the MCPSource marker (the authoritative signal), and expose it on the runtime.ToolExecutor interface. The loop now types tools off that instead of the name shape. Adds TestExecuteNamespacedNonMCPToolNotTypedAsExtension asserting a "__" tool that is not MCP is typed "function" with no mcp.method.name; existing MCP/function span tests updated to drive the marker explicitly. Refs initializ/forge#422 --- forge-core/forgecore_test.go | 2 ++ forge-core/runtime/loop.go | 15 ++++++++---- forge-core/runtime/loop_spans_test.go | 34 +++++++++++++++++++++------ forge-core/runtime/loop_test.go | 7 ++++++ forge-core/tools/registry.go | 17 ++++++++++++++ 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/forge-core/forgecore_test.go b/forge-core/forgecore_test.go index 74c66f6b..0aec7d8b 100644 --- a/forge-core/forgecore_test.go +++ b/forge-core/forgecore_test.go @@ -54,6 +54,8 @@ func (m *mockToolExecutor) Execute(ctx context.Context, name string, arguments j return "", nil } +func (m *mockToolExecutor) IsMCPTool(string) bool { return false } + func (m *mockToolExecutor) ToolDefinitions() []llm.ToolDefinition { var defs []llm.ToolDefinition for name := range m.tools { diff --git a/forge-core/runtime/loop.go b/forge-core/runtime/loop.go index f6337efd..777233c1 100644 --- a/forge-core/runtime/loop.go +++ b/forge-core/runtime/loop.go @@ -41,6 +41,11 @@ const emptyAssistantPlaceholder = "(continuing — previous response was truncat type ToolExecutor interface { Execute(ctx context.Context, name string, arguments json.RawMessage) (string, error) ToolDefinitions() []llm.ToolDefinition + // IsMCPTool reports whether the named tool is backed by an MCP server. + // Authoritative signal for the gen_ai.tool.type / mcp.* span attributes — + // distinct from the "__" name shape, which non-MCP + // namespaced (API per-op) tools also use. + IsMCPTool(name string) bool } // Pre-hook safety ceiling for deferred tool-result truncation: hooks must @@ -787,11 +792,13 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // args/result content capture under CaptureContent + Redact. toolCtx, toolSpan := Tracer().Start(ctx, "tool."+tc.Function.Name) // OTel GenAI tool conventions (gen_ai.tool.*). MCP-backed tools - // are namespaced "__" and map to the "extension" - // tool type (an agent-side bridge to an external system); all - // others are "function". + // map to the "extension" tool type (an agent-side bridge to an + // external system); all others are "function". Uses the + // registry's MCPSource marker, not the "__" name + // shape — non-MCP namespaced (API per-op) tools share that shape + // but must not be typed as MCP extensions. toolType := observability.ToolTypeFunction - isMCPTool := strings.Contains(tc.Function.Name, "__") + isMCPTool := e.tools != nil && e.tools.IsMCPTool(tc.Function.Name) if isMCPTool { toolType = observability.ToolTypeExtension } diff --git a/forge-core/runtime/loop_spans_test.go b/forge-core/runtime/loop_spans_test.go index 3b25be87..54f55bc2 100644 --- a/forge-core/runtime/loop_spans_test.go +++ b/forge-core/runtime/loop_spans_test.go @@ -306,10 +306,10 @@ func TestExecuteRecordsToolErrorOnSpan(t *testing.T) { } // genAIToolRun drives one tool call through the executor and returns the -// recorder so a test can assert span attributes. toolName controls whether -// the tool span is treated as a "function" (builtin/skill) or "extension" -// (MCP-namespaced "__"). -func genAIToolRun(t *testing.T, toolName string) *observability.SpanRecorder { +// recorder so a test can assert span attributes. isMCP mirrors the registry's +// MCPSource marker: it — not the tool name's "__" shape — decides whether the +// span is typed "extension" and carries mcp.method.name. +func genAIToolRun(t *testing.T, toolName string, isMCP bool) *observability.SpanRecorder { t.Helper() tp, rec := observability.NewTestTracerProvider() SetTracerProvider(tp) @@ -351,6 +351,7 @@ func genAIToolRun(t *testing.T, toolName string) *observability.SpanRecorder { executeFunc: func(_ context.Context, _ string, _ json.RawMessage) (string, error) { return "sunny, 20C", nil }, + mcpNames: map[string]bool{toolName: isMCP}, } exec := NewLLMExecutor(LLMExecutorConfig{ Client: client, @@ -372,7 +373,7 @@ func genAIToolRun(t *testing.T, toolName string) *observability.SpanRecorder { // TestExecuteStampsGenAIAgentAndCompletionAttrs asserts the semconv // identity/operation attributes on the agent.execute + llm.completion spans. func TestExecuteStampsGenAIAgentAndCompletionAttrs(t *testing.T) { - rec := genAIToolRun(t, "weather") + rec := genAIToolRun(t, "weather", false) root, ok := rec.FindSpan("agent.execute") if !ok { @@ -412,7 +413,7 @@ func TestExecuteStampsGenAIAgentAndCompletionAttrs(t *testing.T) { // TestExecuteStampsGenAIToolAttrs asserts the semconv gen_ai.tool.* attrs on // a function-tool span, and that mcp.method.name is absent for it. func TestExecuteStampsGenAIToolAttrs(t *testing.T) { - rec := genAIToolRun(t, "weather") + rec := genAIToolRun(t, "weather", false) toolSpan, ok := rec.FindSpan("tool.weather") if !ok { @@ -437,7 +438,7 @@ func TestExecuteStampsGenAIToolAttrs(t *testing.T) { // TestExecuteMCPToolSpanUsesExtensionType asserts an MCP-namespaced tool // ("__") is typed "extension" and carries mcp.method.name. func TestExecuteMCPToolSpanUsesExtensionType(t *testing.T) { - rec := genAIToolRun(t, "linear__create_issue") + rec := genAIToolRun(t, "linear__create_issue", true) toolSpan, ok := rec.FindSpan("tool.linear__create_issue") if !ok { @@ -450,3 +451,22 @@ func TestExecuteMCPToolSpanUsesExtensionType(t *testing.T) { t.Errorf("mcp.method.name = %q (ok=%v); want %q", got, ok, observability.MCPMethodToolsCall) } } + +// TestExecuteNamespacedNonMCPToolNotTypedAsExtension guards the fix for the +// name-heuristic edge: a NON-MCP namespaced tool (API per-op "__") +// shares the "__" shape but must be typed "function" with no mcp.method.name, +// because classification is driven by the registry's MCPSource marker. +func TestExecuteNamespacedNonMCPToolNotTypedAsExtension(t *testing.T) { + rec := genAIToolRun(t, "weatherapi__forecast", false) + + toolSpan, ok := rec.FindSpan("tool.weatherapi__forecast") + if !ok { + t.Fatal("missing tool.weatherapi__forecast span") + } + if got, _ := findAttr(toolSpan, observability.AttrGenAIToolType); got != observability.ToolTypeFunction { + t.Errorf("gen_ai.tool.type = %q; want %q for a non-MCP namespaced tool", got, observability.ToolTypeFunction) + } + if _, ok := findAttr(toolSpan, observability.AttrMCPMethodName); ok { + t.Error("mcp.method.name must not be set on a non-MCP namespaced tool") + } +} diff --git a/forge-core/runtime/loop_test.go b/forge-core/runtime/loop_test.go index 1b58047e..08649476 100644 --- a/forge-core/runtime/loop_test.go +++ b/forge-core/runtime/loop_test.go @@ -30,6 +30,9 @@ func (m *mockLLMClient) ModelID() string { return "test-model" } type mockToolExecutor struct { executeFunc func(ctx context.Context, name string, arguments json.RawMessage) (string, error) toolDefs []llm.ToolDefinition + // mcpNames marks which tool names the executor should treat as + // MCP-backed (mirrors the registry's MCPSource marker). nil => none. + mcpNames map[string]bool } func (m *mockToolExecutor) Execute(ctx context.Context, name string, arguments json.RawMessage) (string, error) { @@ -40,6 +43,10 @@ func (m *mockToolExecutor) ToolDefinitions() []llm.ToolDefinition { return m.toolDefs } +func (m *mockToolExecutor) IsMCPTool(name string) bool { + return m.mcpNames[name] +} + func TestToolResultTruncation(t *testing.T) { // Generate a tool result that exceeds the proportional limit. // With CharBudget=100_000, the tool limit = 25K, so a 60K result gets truncated. diff --git a/forge-core/tools/registry.go b/forge-core/tools/registry.go index 076e46eb..3bfce232 100644 --- a/forge-core/tools/registry.go +++ b/forge-core/tools/registry.go @@ -113,6 +113,23 @@ func (r *Registry) Filter(allowed []string) *Registry { return filtered } +// IsMCPTool reports whether the named tool was discovered from an MCP +// server (implements the MCPSource marker). This is the authoritative +// signal for MCP-backedness — distinct from the "__" name +// shape, which NamespacedSource tools (e.g. per-op API tools "__") +// also use without being MCP. Returns false for unknown or non-MCP tools. +// Satisfies the engine.ToolExecutor interface. +func (r *Registry) IsMCPTool(name string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + t, ok := r.tools[name] + if !ok { + return false + } + _, isMCP := t.(MCPSource) + return isMCP +} + // ToolDefinitions returns LLM tool definitions for all registered tools. // This method satisfies the engine.ToolExecutor interface. func (r *Registry) ToolDefinitions() []llm.ToolDefinition {