diff --git a/docs/core-concepts/observability-tracing.md b/docs/core-concepts/observability-tracing.md index 27dc1b25..2124ca0a 100644 --- a/docs/core-concepts/observability-tracing.md +++ b/docs/core-concepts/observability-tracing.md @@ -212,14 +212,25 @@ Forge mixes OTel GenAI semconv with Forge-specific `forge.*` namespaced attribut | `forge.correlation_id` | `agent.execute` | inbound `X-Forge-Correlation-Id` | | `forge.loop.iteration` | `agent.execute` (set at End) | turn count | | `forge.task.final_state` | `agent.execute` (set at End) | `completed` / `failed` / `canceled` | -| `gen_ai.system` | `agent.execute`, `llm.completion` | `"anthropic"`, `"openai"`, `"ollama"` | -| `gen_ai.request.model` / `.response.model` | `llm.completion` | provider request/response model | +| `gen_ai.provider.name` | `agent.execute`, `llm.completion` | `"anthropic"`, `"openai"`, `"ollama"` — current key (see deprecation note below) | +| `gen_ai.system` | `agent.execute`, `llm.completion` | same value as `gen_ai.provider.name`; **deprecated**, emitted one release for compatibility | +| `gen_ai.operation.name` | `llm.completion` (`chat`), `tool.` (`execute_tool`) | operation kind | +| `gen_ai.agent.id` / `.name` / `.version` | `agent.execute` | `forge.yaml` `agent_id` (id + name) / `version` | +| `gen_ai.conversation.id` | `agent.execute` | Forge session id (A2A task id) | +| `gen_ai.request.model` | `agent.execute`, `llm.completion` | requested model | +| `gen_ai.response.model` | `llm.completion` | vendor-reported model (falls back to request model) | +| `gen_ai.response.id` | `llm.completion` | provider completion id | | `gen_ai.usage.input_tokens` / `.output_tokens` | `llm.completion` | provider usage block | | `gen_ai.response.finish_reasons` | `llm.completion` | provider stop reason | -| `forge.tool.name` | `tool.` | tool function name | -| `forge.tool.error` | `tool.` | error message on failure | +| `gen_ai.tool.name` | `tool.` | tool function name | +| `gen_ai.tool.call.id` | `tool.` | LLM-assigned tool-call id | +| `gen_ai.tool.type` | `tool.` | `function` (builtin/skill) or `extension` (MCP-backed) | +| `mcp.method.name` | `tool.` (MCP only) | `tools/call` | +| `error.type` | `tool.` (on failure) | `tool_execution_error` | -Tool errors do **not** fail the outer `agent.execute` span — they surface to the LLM as text and the loop continues. The tool span carries the failure detail so operators can pivot from a trace to the specific failed invocation. +Tool spans follow the OTel **GenAI semantic conventions** (`gen_ai.tool.*`) — these replaced the former proprietary `forge.tool.*` keys, which never shipped to production. MCP-backed tools (namespaced `__`) are typed `extension` and additionally carry `mcp.method.name=tools/call`. (`mcp.session.id` / `mcp.protocol.version` require plumbing the MCP manager to the executor and are tracked as a follow-up.) + +Tool errors do **not** fail the outer `agent.execute` span — they surface to the LLM as text and the loop continues. The tool span carries the failure detail (`error.type` + span status `Error`) so operators can pivot from a trace to the specific failed invocation. ### Span content capture @@ -227,18 +238,20 @@ Prompts, completions, tool args, and tool results are **off by default** — Pha | `forge.yaml` knob | Span | Attribute keys added when `capture_content: true` | |---|---|---| -| (always) | `llm.completion` | `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.response.finish_reasons` | +| (always) | `agent.execute` | `gen_ai.provider.name`, `gen_ai.agent.id`, `gen_ai.agent.name`, `gen_ai.agent.version`, `gen_ai.conversation.id`, `gen_ai.request.model` | +| `capture_content: true` | `agent.execute` | `gen_ai.tool.definitions` (JSON array of the tool catalog available to the agent — potentially large, hence opt-in) | +| (always) | `llm.completion` | `gen_ai.operation.name` (`chat`), `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`, `gen_ai.response.id`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.response.finish_reasons` | | `capture_content: true` | `llm.completion` | `gen_ai.input.messages` (JSON array of role+content pairs sent to the model), `gen_ai.output.messages` (JSON single-element array of role+content for the model's response) — current OTel GenAI semconv, supersedes the deprecated flat-string `gen_ai.prompt` / `gen_ai.completion` | -| (always) | `tool.` | `forge.tool.name`, `forge.tool.error` (on failure) | -| `capture_content: true` | `tool.` | `forge.tool.args` (raw arguments JSON), `forge.tool.result` (raw output) | +| (always) | `tool.` | `gen_ai.operation.name` (`execute_tool`), `gen_ai.tool.name`, `gen_ai.tool.call.id`, `gen_ai.tool.type`, `mcp.method.name` (MCP only), `error.type` (on failure) | +| `capture_content: true` | `tool.` | `gen_ai.tool.call.arguments` (raw arguments JSON), `gen_ai.tool.call.result` (raw output), `gen_ai.tool.description` (from the tool definition) | When `capture_content: true` and `redact: true` (the default when capture is on), attribute values pass through a redactor that scrubs the same vendor secret-token shapes the runtime guardrails default rules cover (Anthropic `sk-ant-…`, OpenAI `sk-…`, GitHub `ghp_/gho_/ghs_/github_pat_…`, AWS `AKIA…`, Slack `xoxb-/xoxp-…`, RSA/EC/OPENSSH/PRIVATE key blocks, Telegram bot tokens). Matched values become `[REDACTED]`. Setting `redact: false` is the enterprise raw-capture path — content is stamped verbatim with the byte cap still applied. Every captured value is byte-capped at **4 KiB** (below the 5 KiB attribute soft-cap most backends apply). When the input exceeds the cap, the value ends with a `…[truncated:N]` marker where `N` is the original byte length. The marker is **byte-identical** to what the audit payload-capture path emits for the same input, so an operator grepping `[truncated:` across span attributes and audit rows sees aligned output. -**Default posture** (no opt-in): the `gen_ai.input.messages`, `gen_ai.output.messages`, `forge.tool.args`, `forge.tool.result` keys are **absent** from spans — not set to empty string. Backends that gate dashboards on "is this key present?" can distinguish "metadata-only by default" from "operator opted in but the field happened to be empty." +**Default posture** (no opt-in): the `gen_ai.input.messages`, `gen_ai.output.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, `gen_ai.tool.description`, and `gen_ai.tool.definitions` keys are **absent** from spans — not set to empty string. Backends that gate dashboards on "is this key present?" can distinguish "metadata-only by default" from "operator opted in but the field happened to be empty." -**OTel semconv versioning note**: the GenAI semantic conventions moved from flat-string (`gen_ai.prompt`, `gen_ai.completion`) to structured (`gen_ai.input.messages`, `gen_ai.output.messages`) attributes. Forge emits only the **current** structured keys. Backends that only recognize the deprecated flat-string attributes will not show prompt / completion text on Forge spans — upgrade the backend's semconv mapping or use a span processor to translate. +**OTel semconv versioning note**: the GenAI semantic conventions moved from flat-string (`gen_ai.prompt`, `gen_ai.completion`) to structured (`gen_ai.input.messages`, `gen_ai.output.messages`) attributes, and renamed `gen_ai.system` → `gen_ai.provider.name`. Forge emits the **current** structured keys and `gen_ai.provider.name`, and continues to emit the deprecated `gen_ai.system` for one release for compatibility. Backends that only recognize the older attributes should upgrade their semconv mapping or use a span processor to translate. ### Guardrail spans (issue #161) @@ -268,7 +281,7 @@ Attribute reference: **Default posture**: `forge.guardrail.evidence` is absent unless `capture_content: true`. The other five attributes are always present when a gate fires (cheap, no PII risk). When tracing is disabled, the noop tracer short-circuits and the spans are not produced at all. -**Content-capture parity**: the evidence attribute uses the exact same `PrepareSpanContent(redact, maxBytes)` pipeline as `gen_ai.input.messages` and `forge.tool.args` — same vendor secret-token scrub, same 4 KiB byte cap, same `…[truncated:N]` marker. Operators get one mental model across all four content streams (LLM input / LLM output / tool args / tool result / guardrail evidence). +**Content-capture parity**: the evidence attribute uses the exact same `PrepareSpanContent(redact, maxBytes)` pipeline as `gen_ai.input.messages` and `gen_ai.tool.call.arguments` — same vendor secret-token scrub, same 4 KiB byte cap, same `…[truncated:N]` marker. Operators get one mental model across all four content streams (LLM input / LLM output / tool args / tool result / guardrail evidence). ## End-to-end propagation (Phase 5) diff --git a/forge-cli/runtime/runner.go b/forge-cli/runtime/runner.go index 2d55484a..c3ffee17 100644 --- a/forge-cli/runtime/runner.go +++ b/forge-cli/runtime/runner.go @@ -1337,6 +1337,8 @@ func (r *Runner) Run(ctx context.Context) error { Logger: r.logger, ModelName: mc.Client.Model, Provider: mc.Provider, + AgentID: r.cfg.Config.AgentID, + AgentVersion: r.cfg.Config.Version, MaxIterations: 100, CharBudget: charBudget, FilesDir: filepath.Join(r.cfg.WorkDir, ".forge", "files"), diff --git a/forge-core/forgecore_test.go b/forge-core/forgecore_test.go index 74c66f6b..0aec7d8b 100644 --- a/forge-core/forgecore_test.go +++ b/forge-core/forgecore_test.go @@ -54,6 +54,8 @@ func (m *mockToolExecutor) Execute(ctx context.Context, name string, arguments j return "", nil } +func (m *mockToolExecutor) IsMCPTool(string) bool { return false } + func (m *mockToolExecutor) ToolDefinitions() []llm.ToolDefinition { var defs []llm.ToolDefinition for name := range m.tools { diff --git a/forge-core/llm/providers/anthropic.go b/forge-core/llm/providers/anthropic.go index ea3f6304..d4fa2a09 100644 --- a/forge-core/llm/providers/anthropic.go +++ b/forge-core/llm/providers/anthropic.go @@ -333,6 +333,7 @@ func (c *AnthropicClient) convertMessage(m llm.ChatMessage) anthropicMessage { // Anthropic-specific response types. type anthropicResponse struct { ID string `json:"id"` + Model string `json:"model"` Content []anthropicContentBlock `json:"content"` StopReason string `json:"stop_reason"` Usage struct { @@ -375,6 +376,7 @@ func (c *AnthropicClient) parseAnthropicResponse(body io.Reader) (*llm.ChatRespo return &llm.ChatResponse{ ID: resp.ID, + Model: resp.Model, Message: msg, Usage: llm.UsageInfo{ InputTokens: resp.Usage.InputTokens, diff --git a/forge-core/llm/providers/openai.go b/forge-core/llm/providers/openai.go index 92f743c9..bba1bb28 100644 --- a/forge-core/llm/providers/openai.go +++ b/forge-core/llm/providers/openai.go @@ -257,6 +257,7 @@ func derivePromptCacheKey(model string, req *llm.ChatRequest) string { // openaiResponse is the OpenAI-specific response format. type openaiResponse struct { ID string `json:"id"` + Model string `json:"model"` Choices []struct { Message struct { Role string `json:"role"` @@ -284,7 +285,8 @@ func (c *OpenAIClient) parseOpenAIResponse(body io.Reader) (*llm.ChatResponse, e choice := resp.Choices[0] return &llm.ChatResponse{ - ID: resp.ID, + ID: resp.ID, + Model: resp.Model, Message: llm.ChatMessage{ Role: choice.Message.Role, Content: choice.Message.Content, diff --git a/forge-core/llm/types.go b/forge-core/llm/types.go index ebb9a6bb..04fb5182 100644 --- a/forge-core/llm/types.go +++ b/forge-core/llm/types.go @@ -64,6 +64,13 @@ type ChatResponse struct { Message ChatMessage `json:"message"` Usage UsageInfo `json:"usage"` FinishReason string `json:"finish_reason"` + // Model is the model the provider REPORTED generating the response, + // parsed from the provider response body when present (Anthropic / + // OpenAI both echo "model"). Often identical to the request model but + // enterprise routers can substitute a versioned suffix. Surfaced as + // the gen_ai.response.model span attribute; empty when the provider + // does not report it (the executor falls back to the request model). + Model string `json:"model,omitempty"` // Endpoint is the URL the client POSTed to (base URL + provider path). // Set by the provider client so the llm_call audit event can record the // invoked path even when payload capture is off. Internal only (json:"-"). diff --git a/forge-core/observability/attrs.go b/forge-core/observability/attrs.go index e8602015..de952156 100644 --- a/forge-core/observability/attrs.go +++ b/forge-core/observability/attrs.go @@ -24,9 +24,16 @@ package observability // "llm.completion") and reads naturally inline. const ( - // ─── GenAI semconv (draft, pinned to OTel semconv 1.26.0 GenAI). ── + // ─── GenAI semconv (draft). ────────────────────────────────────── // Backends like Honeycomb / Datadog / Grafana Tempo group LLM // activity by these. Naming follows the OTel GenAI spec exactly. + // The core token/model keys below track the 1.26.0 snapshot; the + // agent / operation / tool keys added later (provider.name, + // operation.name, agent.*, conversation.id, tool.*) track the newer + // GenAI registry. Because these are hand-declared string constants + // (not imported from the semconv package), a spec bump is this + // one-file sweep — the resource-level semconv import in otel.go is + // versioned separately and only carries service.* + schema URL. // AttrGenAISystem identifies the LLM vendor: "anthropic", // "openai", "ollama", "openai-compatible". @@ -51,6 +58,70 @@ const ( // "max_tokens", "end_turn", etc. AttrGenAIResponseFinishReasons = "gen_ai.response.finish_reasons" + // AttrGenAIProviderName is the current OTel key for the GenAI vendor + // ("anthropic", "openai", "ollama", ...). It supersedes the + // deprecated AttrGenAISystem (`gen_ai.system`); Forge emits BOTH for + // one release so dashboards keyed on either light up, then drops the + // alias. Same value as AttrGenAISystem. + AttrGenAIProviderName = "gen_ai.provider.name" + + // AttrGenAIOperationName identifies the operation the span measures — + // "chat" on the llm.completion span, "execute_tool" on a tool. + // span. Backends group GenAI activity by (operation.name, provider.name). + AttrGenAIOperationName = "gen_ai.operation.name" + + // OpChat / OpExecuteTool are the two AttrGenAIOperationName values + // Forge emits today. + OpChat = "chat" + OpExecuteTool = "execute_tool" + + // AttrGenAIResponseID is the provider's completion id (Anthropic + // "msg_…", OpenAI "chatcmpl-…"). Read straight from ChatResponse.ID. + AttrGenAIResponseID = "gen_ai.response.id" + + // AttrGenAIConversationID is the stable conversation/thread id. Forge + // maps this to the A2A task id — the session-store key that persists a + // transcript across turns (.forge/sessions/.json). Per semconv + // this MUST be a real thread identifier, never a synthesized UUID or + // trace id; the Forge session id qualifies. + AttrGenAIConversationID = "gen_ai.conversation.id" + + // AttrGenAIAgent* stamp the agent's identity on the agent.execute + // span from forge.yaml. `agent_id` doubles as the human-readable name + // (forge.yaml has no separate name field, matching the A2A card's + // AgentID fallback), so id and name carry the same value today. + AttrGenAIAgentID = "gen_ai.agent.id" + AttrGenAIAgentName = "gen_ai.agent.name" + AttrGenAIAgentVersion = "gen_ai.agent.version" + + // AttrGenAITool* name the tool-call instrumentation on tool. + // spans, following the OTel GenAI tool conventions. These REPLACE the + // former proprietary `forge.tool.*` keys (which never shipped to + // production). Content-bearing tool attributes (arguments / result / + // description / definitions) live in the content-capture block below. + AttrGenAIToolName = "gen_ai.tool.name" + AttrGenAIToolCallID = "gen_ai.tool.call.id" + AttrGenAIToolType = "gen_ai.tool.type" + + // ToolTypeFunction / ToolTypeExtension are the AttrGenAIToolType + // values Forge emits: builtin + skill tools are "function"; MCP-backed + // tools (namespaced "__") are "extension" — an + // agent-side bridge to an external system. + ToolTypeFunction = "function" + ToolTypeExtension = "extension" + + // AttrMCPMethodName is the MCP JSON-RPC method a span represents. For a + // Forge tool call backed by an MCP server it is always "tools/call". + // (mcp.session.id / mcp.protocol.version require plumbing the MCP + // manager down to the executor and are tracked as a follow-up.) + AttrMCPMethodName = "mcp.method.name" + MCPMethodToolsCall = "tools/call" + + // AttrErrorType is the OTel-standard error classification set on a + // tool. span when execution fails (alongside RecordError + + // Status=Error). Replaces the former `forge.tool.error`. + AttrErrorType = "error.type" + // ─── Forge-specific attributes. ────────────────────────────────── // AttrForgeAgentID is the agent_id from forge.yaml — the operator's @@ -93,10 +164,12 @@ const ( // dashboards can chart "iterations per task." AttrForgeLoopIteration = "forge.loop.iteration" - // AttrForgeToolName / AttrForgeToolError name the tool call - // instrumentation. - AttrForgeToolName = "forge.tool.name" - AttrForgeToolError = "forge.tool.error" + // AttrForgeToolName cross-references which tool a guardrail. + // span evaluated (see forge-cli/runtime/guardrails_tracing.go). The + // tool. execution span itself uses the semconv AttrGenAIToolName + // key instead. (The former forge.tool.error was removed in favor of the + // standard AttrErrorType.) + AttrForgeToolName = "forge.tool.name" // AttrForgeTaskFinalState is the terminal A2A TaskState the loop // resolved to — "completed", "failed", "canceled". Set on the @@ -125,13 +198,26 @@ const ( // `gen_ai.completion` flat-string attribute. AttrGenAIOutputMessages = "gen_ai.output.messages" - // AttrForgeToolArgs is the raw arguments JSON the agent passed to - // a tool. Set on tool. spans. - AttrForgeToolArgs = "forge.tool.args" - - // AttrForgeToolResult is the raw output the tool returned. Set on - // tool. spans. - AttrForgeToolResult = "forge.tool.result" + // AttrGenAIToolCallArguments is the raw arguments JSON the agent passed + // to a tool (semconv `gen_ai.tool.call.arguments`). Set on tool. + // spans. Replaces the former `forge.tool.args`. + AttrGenAIToolCallArguments = "gen_ai.tool.call.arguments" + + // AttrGenAIToolCallResult is the raw output the tool returned (semconv + // `gen_ai.tool.call.result`). Set on tool. spans. Replaces the + // former `forge.tool.result`. + AttrGenAIToolCallResult = "gen_ai.tool.call.result" + + // AttrGenAIToolDescription is the tool's description from its + // definition (semconv `gen_ai.tool.description`). Flagged sensitive by + // semconv, so it rides the same CaptureContent opt-in. + AttrGenAIToolDescription = "gen_ai.tool.description" + + // AttrGenAIToolDefinitions is the JSON array of tool definitions + // available to the agent (semconv `gen_ai.tool.definitions`), stamped + // once on the agent.execute span. Potentially large, so it is opt-in + // behind CaptureContent. + AttrGenAIToolDefinitions = "gen_ai.tool.definitions" // ─── Guardrail span attributes (issue #161) ────────────────────── // diff --git a/forge-core/runtime/loop.go b/forge-core/runtime/loop.go index 7be7a16a..777233c1 100644 --- a/forge-core/runtime/loop.go +++ b/forge-core/runtime/loop.go @@ -41,6 +41,11 @@ const emptyAssistantPlaceholder = "(continuing — previous response was truncat type ToolExecutor interface { Execute(ctx context.Context, name string, arguments json.RawMessage) (string, error) ToolDefinitions() []llm.ToolDefinition + // IsMCPTool reports whether the named tool is backed by an MCP server. + // Authoritative signal for the gen_ai.tool.type / mcp.* span attributes — + // distinct from the "__" name shape, which non-MCP + // namespaced (API per-op) tools also use. + IsMCPTool(name string) bool } // Pre-hook safety ceiling for deferred tool-result truncation: hooks must @@ -94,6 +99,8 @@ type LLMExecutor struct { logger Logger modelName string // resolved model name for context budget provider string // resolved provider name (anthropic, openai, ollama, custom) + agentID string // forge.yaml agent_id — gen_ai.agent.id / .name + agentVersion string // forge.yaml version — gen_ai.agent.version charBudget int // resolved character budget maxToolResultChars int // computed from char budget filesDir string // directory for file_create output @@ -126,6 +133,8 @@ type LLMExecutorConfig struct { Logger Logger ModelName string // model name for context-aware budgeting Provider string // provider name (anthropic, openai, ollama, custom) — for audit attribution + AgentID string // forge.yaml agent_id — stamped as gen_ai.agent.id / gen_ai.agent.name on the agent.execute span + AgentVersion string // forge.yaml version — stamped as gen_ai.agent.version CharBudget int // explicit char budget override (0 = auto from model) FilesDir string // directory for file_create output (default: $TMPDIR/forge-files) SessionMaxAge time.Duration // max idle time before session recovery is skipped (0 = 30m default) @@ -194,6 +203,8 @@ func NewLLMExecutor(cfg LLMExecutorConfig) *LLMExecutor { logger: logger, modelName: cfg.ModelName, provider: cfg.Provider, + agentID: cfg.AgentID, + agentVersion: cfg.AgentVersion, charBudget: budget, maxToolResultChars: toolLimit, filesDir: cfg.FilesDir, @@ -242,16 +253,37 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess }() if task != nil && task.ID != "" { span.SetAttributes(attribute.String(observability.AttrForgeTaskID, task.ID)) + // gen_ai.conversation.id — the Forge session id (A2A task id) is + // the stable thread key that persists a transcript across turns, + // exactly what semconv wants here. + span.SetAttributes(attribute.String(observability.AttrGenAIConversationID, task.ID)) } if cid := CorrelationIDFromContext(ctx); cid != "" { span.SetAttributes(attribute.String(observability.AttrForgeCorrelationID, cid)) } if e.provider != "" { - span.SetAttributes(attribute.String(observability.AttrGenAISystem, e.provider)) + // gen_ai.system is the deprecated key; gen_ai.provider.name is the + // current one. Emit both for one release. + span.SetAttributes( + attribute.String(observability.AttrGenAISystem, e.provider), + attribute.String(observability.AttrGenAIProviderName, e.provider), + ) } if e.modelName != "" { span.SetAttributes(attribute.String(observability.AttrGenAIRequestModel, e.modelName)) } + // gen_ai.agent.* — identity from forge.yaml. agent_id doubles as the + // human-readable name (no separate name field), so id and name carry the + // same value. + if e.agentID != "" { + span.SetAttributes( + attribute.String(observability.AttrGenAIAgentID, e.agentID), + attribute.String(observability.AttrGenAIAgentName, e.agentID), + ) + } + if e.agentVersion != "" { + span.SetAttributes(attribute.String(observability.AttrGenAIAgentVersion, e.agentVersion)) + } mem := NewMemory(e.systemPrompt, e.charBudget, e.modelName) @@ -343,6 +375,26 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess toolDefs = e.tools.ToolDefinitions() } + // Map tool name → description for the gen_ai.tool.description attribute + // on per-call tool spans below. + toolDescByName := make(map[string]string, len(toolDefs)) + for _, td := range toolDefs { + toolDescByName[td.Function.Name] = td.Function.Description + } + + // gen_ai.tool.definitions — the full tool catalog available to the + // agent, stamped once on the agent.execute span. Opt-in (CaptureContent) + // because it can be large; runs through the same redact-then-truncate + // pipeline as the other content attributes. + if e.tracingCfg.CaptureContent && len(toolDefs) > 0 { + if defs, err := json.Marshal(toolDefs); err == nil { + span.SetAttributes(attribute.String( + observability.AttrGenAIToolDefinitions, + PrepareSpanContent(string(defs), e.tracingCfg.Redact, DefaultSpanContentCapBytes), + )) + } + } + // Track large tool outputs so they can be included as file parts // in the response (the LLM may truncate them due to output token limits). var largeToolOutputs []a2a.Part @@ -424,7 +476,9 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // not part of the LLM call's wall-clock measurement. llmCtx, llmSpan := Tracer().Start(ctx, "llm.completion") llmSpan.SetAttributes( + attribute.String(observability.AttrGenAIOperationName, observability.OpChat), attribute.String(observability.AttrGenAISystem, e.provider), + attribute.String(observability.AttrGenAIProviderName, e.provider), attribute.String(observability.AttrGenAIRequestModel, e.modelName), ) // Phase 3.5 (#130) — stamp the structured input messages on the @@ -478,6 +532,18 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess attribute.Int(observability.AttrGenAIUsageInputTokens, resp.Usage.InputTokens), attribute.Int(observability.AttrGenAIUsageOutputTokens, resp.Usage.OutputTokens), ) + if resp.ID != "" { + llmSpan.SetAttributes(attribute.String(observability.AttrGenAIResponseID, resp.ID)) + } + // gen_ai.response.model — the model the vendor reported back. Falls + // back to the request model when the provider does not echo one. + respModel := resp.Model + if respModel == "" { + respModel = e.modelName + } + if respModel != "" { + llmSpan.SetAttributes(attribute.String(observability.AttrGenAIResponseModel, respModel)) + } if resp.FinishReason != "" { llmSpan.SetAttributes(attribute.StringSlice(observability.AttrGenAIResponseFinishReasons, []string{resp.FinishReason})) } @@ -725,7 +791,31 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // by kind without a query. Phase 3.5 (#130) added optional // args/result content capture under CaptureContent + Redact. toolCtx, toolSpan := Tracer().Start(ctx, "tool."+tc.Function.Name) - toolSpan.SetAttributes(attribute.String(observability.AttrForgeToolName, tc.Function.Name)) + // OTel GenAI tool conventions (gen_ai.tool.*). MCP-backed tools + // map to the "extension" tool type (an agent-side bridge to an + // external system); all others are "function". Uses the + // registry's MCPSource marker, not the "__" name + // shape — non-MCP namespaced (API per-op) tools share that shape + // but must not be typed as MCP extensions. + toolType := observability.ToolTypeFunction + isMCPTool := e.tools != nil && e.tools.IsMCPTool(tc.Function.Name) + if isMCPTool { + toolType = observability.ToolTypeExtension + } + toolSpan.SetAttributes( + attribute.String(observability.AttrGenAIOperationName, observability.OpExecuteTool), + attribute.String(observability.AttrGenAIToolName, tc.Function.Name), + attribute.String(observability.AttrGenAIToolType, toolType), + ) + if tc.ID != "" { + toolSpan.SetAttributes(attribute.String(observability.AttrGenAIToolCallID, tc.ID)) + } + // MCP tool calls carry the MCP method that the executor issues + // downstream. (mcp.session.id / mcp.protocol.version need the MCP + // manager plumbed to the executor — tracked as a follow-up.) + if isMCPTool { + toolSpan.SetAttributes(attribute.String(observability.AttrMCPMethodName, observability.MCPMethodToolsCall)) + } // With compression enabled (deferToolTruncation), relax // tool-internal output caps so the full result reaches the // compression hook instead of being destructively cut inside @@ -733,18 +823,30 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess if e.deferToolTruncation { toolCtx = tools.WithRelaxedLimits(toolCtx) } - if e.tracingCfg.CaptureContent && tc.Function.Arguments != "" { - toolSpan.SetAttributes(attribute.String( - observability.AttrForgeToolArgs, - PrepareSpanContent(tc.Function.Arguments, e.tracingCfg.Redact, DefaultSpanContentCapBytes), - )) + // gen_ai.tool.description + gen_ai.tool.call.arguments are + // content-bearing; semconv flags them sensitive, so they ride + // the CaptureContent opt-in. + if e.tracingCfg.CaptureContent { + if desc := toolDescByName[tc.Function.Name]; desc != "" { + toolSpan.SetAttributes(attribute.String( + observability.AttrGenAIToolDescription, + PrepareSpanContent(desc, e.tracingCfg.Redact, DefaultSpanContentCapBytes), + )) + } + if tc.Function.Arguments != "" { + toolSpan.SetAttributes(attribute.String( + observability.AttrGenAIToolCallArguments, + PrepareSpanContent(tc.Function.Arguments, e.tracingCfg.Redact, DefaultSpanContentCapBytes), + )) + } } result, execErr := e.tools.Execute(toolCtx, tc.Function.Name, json.RawMessage(tc.Function.Arguments)) toolDuration := time.Since(toolStart) if execErr != nil { toolSpan.RecordError(execErr) toolSpan.SetStatus(codes.Error, execErr.Error()) - toolSpan.SetAttributes(attribute.String(observability.AttrForgeToolError, execErr.Error())) + // OTel-standard error classification (replaces forge.tool.error). + toolSpan.SetAttributes(attribute.String(observability.AttrErrorType, "tool_execution_error")) result = fmt.Sprintf("Error executing tool %s: %s", tc.Function.Name, execErr.Error()) } // Phase 3.5 (#130) — tool result content capture. The @@ -754,7 +856,7 @@ func (e *LLMExecutor) Execute(ctx context.Context, task *a2a.Task, msg *a2a.Mess // will see on the next iteration. if e.tracingCfg.CaptureContent && result != "" { toolSpan.SetAttributes(attribute.String( - observability.AttrForgeToolResult, + observability.AttrGenAIToolCallResult, PrepareSpanContent(result, e.tracingCfg.Redact, DefaultSpanContentCapBytes), )) } diff --git a/forge-core/runtime/loop_spans_content_test.go b/forge-core/runtime/loop_spans_content_test.go index 24cd7abe..4e8ab415 100644 --- a/forge-core/runtime/loop_spans_content_test.go +++ b/forge-core/runtime/loop_spans_content_test.go @@ -284,34 +284,34 @@ func runOneToolCall(t *testing.T, tracingCfg observability.TracingConfig, toolAr // TestExecute_CaptureContentTrue_StampsToolArgsAndResult exercises the // tool-side mirror of the LLM-span content tests. Args from the -// LLM-emitted tool call land at forge.tool.args; tool stdout/return -// lands at forge.tool.result; both go through the same redact + -// truncate pipeline. +// LLM-emitted tool call land at gen_ai.tool.call.arguments; tool +// stdout/return lands at gen_ai.tool.call.result; both go through the +// same redact + truncate pipeline. func TestExecute_CaptureContentTrue_StampsToolArgsAndResult(t *testing.T) { cfg := observability.TracingConfig{CaptureContent: true, Redact: true} span := runOneToolCall(t, cfg, `{"target":"`+awsKeyFixture+`"}`, "deleted "+awsKeyFixture) - args, ok := findAttr(span, observability.AttrForgeToolArgs) + args, ok := findAttr(span, observability.AttrGenAIToolCallArguments) if !ok { - t.Error("forge.tool.args missing when CaptureContent=true") + t.Error("gen_ai.tool.call.arguments missing when CaptureContent=true") } else { if strings.Contains(args, awsKeyFixture) { - t.Errorf("raw AWS key survived redaction on forge.tool.args: %q", args) + t.Errorf("raw AWS key survived redaction on gen_ai.tool.call.arguments: %q", args) } if !strings.Contains(args, RedactionMarker) { - t.Errorf("expected redaction marker in forge.tool.args; got %q", args) + t.Errorf("expected redaction marker in gen_ai.tool.call.arguments; got %q", args) } } - result, ok := findAttr(span, observability.AttrForgeToolResult) + result, ok := findAttr(span, observability.AttrGenAIToolCallResult) if !ok { - t.Error("forge.tool.result missing when CaptureContent=true") + t.Error("gen_ai.tool.call.result missing when CaptureContent=true") } else { if strings.Contains(result, awsKeyFixture) { - t.Errorf("raw AWS key survived redaction on forge.tool.result: %q", result) + t.Errorf("raw AWS key survived redaction on gen_ai.tool.call.result: %q", result) } if !strings.Contains(result, RedactionMarker) { - t.Errorf("expected redaction marker in forge.tool.result; got %q", result) + t.Errorf("expected redaction marker in gen_ai.tool.call.result; got %q", result) } } } @@ -322,10 +322,10 @@ func TestExecute_CaptureContentFalse_NoToolContentAttributes(t *testing.T) { cfg := observability.TracingConfig{CaptureContent: false} span := runOneToolCall(t, cfg, `{"target":"foo"}`, "ok") - if _, ok := findAttr(span, observability.AttrForgeToolArgs); ok { - t.Errorf("CaptureContent=false must not set forge.tool.args") + if _, ok := findAttr(span, observability.AttrGenAIToolCallArguments); ok { + t.Errorf("CaptureContent=false must not set gen_ai.tool.call.arguments") } - if _, ok := findAttr(span, observability.AttrForgeToolResult); ok { - t.Errorf("CaptureContent=false must not set forge.tool.result") + if _, ok := findAttr(span, observability.AttrGenAIToolCallResult); ok { + t.Errorf("CaptureContent=false must not set gen_ai.tool.call.result") } } diff --git a/forge-core/runtime/loop_spans_test.go b/forge-core/runtime/loop_spans_test.go index 1681f53c..54f55bc2 100644 --- a/forge-core/runtime/loop_spans_test.go +++ b/forge-core/runtime/loop_spans_test.go @@ -288,12 +288,12 @@ func TestExecuteRecordsToolErrorOnSpan(t *testing.T) { } gotErrAttr := false for _, kv := range toolSpan.Attributes() { - if string(kv.Key) == observability.AttrForgeToolError { + if string(kv.Key) == observability.AttrErrorType { gotErrAttr = true } } if !gotErrAttr { - t.Errorf("tool.broken missing %q attribute", observability.AttrForgeToolError) + t.Errorf("tool.broken missing %q attribute", observability.AttrErrorType) } // Outer span is "completed" — tool errors aren't fatal. @@ -304,3 +304,169 @@ func TestExecuteRecordsToolErrorOnSpan(t *testing.T) { } } } + +// genAIToolRun drives one tool call through the executor and returns the +// recorder so a test can assert span attributes. isMCP mirrors the registry's +// MCPSource marker: it — not the tool name's "__" shape — decides whether the +// span is typed "extension" and carries mcp.method.name. +func genAIToolRun(t *testing.T, toolName string, isMCP bool) *observability.SpanRecorder { + t.Helper() + tp, rec := observability.NewTestTracerProvider() + SetTracerProvider(tp) + t.Cleanup(func() { + ResetTracerProviderForTest() + _ = tp.Shutdown(context.Background()) + }) + + call := 0 + client := &mockLLMClient{ + chatFunc: func(_ context.Context, _ *llm.ChatRequest) (*llm.ChatResponse, error) { + call++ + if call == 1 { + return &llm.ChatResponse{ + ID: "resp-1", + Model: "claude-sonnet-4-6-20990101", // vendor-reported, differs from request model + Message: llm.ChatMessage{ + Role: llm.RoleAssistant, + ToolCalls: []llm.ToolCall{{ + ID: "tc-42", + Type: "function", + Function: llm.FunctionCall{Name: toolName, Arguments: `{"city":"Tokyo"}`}, + }}, + }, + Usage: llm.UsageInfo{InputTokens: 50, OutputTokens: 10}, + FinishReason: "tool_calls", + }, nil + } + return &llm.ChatResponse{ + ID: "resp-2", + Model: "claude-sonnet-4-6-20990101", + Message: llm.ChatMessage{Role: llm.RoleAssistant, Content: "sunny"}, + Usage: llm.UsageInfo{InputTokens: 60, OutputTokens: 3}, + FinishReason: "stop", + }, nil + }, + } + tools := &mockToolExecutor{ + executeFunc: func(_ context.Context, _ string, _ json.RawMessage) (string, error) { + return "sunny, 20C", nil + }, + mcpNames: map[string]bool{toolName: isMCP}, + } + exec := NewLLMExecutor(LLMExecutorConfig{ + Client: client, + Tools: tools, + MaxIterations: 3, + ModelName: "claude-req", + Provider: "anthropic", + AgentID: "weatherbot", + AgentVersion: "1.2.3", + }) + if _, err := exec.Execute(context.Background(), + &a2a.Task{ID: "task-genai"}, + &a2a.Message{Role: a2a.MessageRoleUser, Parts: []a2a.Part{{Kind: a2a.PartKindText, Text: "hi"}}}); err != nil { + t.Fatalf("Execute: %v", err) + } + return rec +} + +// TestExecuteStampsGenAIAgentAndCompletionAttrs asserts the semconv +// identity/operation attributes on the agent.execute + llm.completion spans. +func TestExecuteStampsGenAIAgentAndCompletionAttrs(t *testing.T) { + rec := genAIToolRun(t, "weather", false) + + root, ok := rec.FindSpan("agent.execute") + if !ok { + t.Fatal("missing agent.execute span") + } + wantRoot := map[string]string{ + observability.AttrGenAIProviderName: "anthropic", + observability.AttrGenAIAgentID: "weatherbot", + observability.AttrGenAIAgentName: "weatherbot", + observability.AttrGenAIAgentVersion: "1.2.3", + observability.AttrGenAIConversationID: "task-genai", + } + for k, want := range wantRoot { + if got, ok := findAttr(root, k); !ok || got != want { + t.Errorf("agent.execute %s = %q (ok=%v); want %q", k, got, ok, want) + } + } + + llmSpan, ok := rec.FindSpan("llm.completion") + if !ok { + t.Fatal("missing llm.completion span") + } + if got, _ := findAttr(llmSpan, observability.AttrGenAIOperationName); got != observability.OpChat { + t.Errorf("llm.completion operation.name = %q; want %q", got, observability.OpChat) + } + if got, _ := findAttr(llmSpan, observability.AttrGenAIProviderName); got != "anthropic" { + t.Errorf("llm.completion provider.name = %q; want anthropic", got) + } + if got, ok := findAttr(llmSpan, observability.AttrGenAIResponseModel); !ok || got != "claude-sonnet-4-6-20990101" { + t.Errorf("llm.completion response.model = %q (ok=%v); want the vendor-reported model", got, ok) + } + if got, _ := findAttr(llmSpan, observability.AttrGenAIResponseID); got == "" { + t.Error("llm.completion missing gen_ai.response.id") + } +} + +// TestExecuteStampsGenAIToolAttrs asserts the semconv gen_ai.tool.* attrs on +// a function-tool span, and that mcp.method.name is absent for it. +func TestExecuteStampsGenAIToolAttrs(t *testing.T) { + rec := genAIToolRun(t, "weather", false) + + toolSpan, ok := rec.FindSpan("tool.weather") + if !ok { + t.Fatal("missing tool.weather span") + } + want := map[string]string{ + observability.AttrGenAIOperationName: observability.OpExecuteTool, + observability.AttrGenAIToolName: "weather", + observability.AttrGenAIToolCallID: "tc-42", + observability.AttrGenAIToolType: observability.ToolTypeFunction, + } + for k, wantVal := range want { + if got, ok := findAttr(toolSpan, k); !ok || got != wantVal { + t.Errorf("tool.weather %s = %q (ok=%v); want %q", k, got, ok, wantVal) + } + } + if _, ok := findAttr(toolSpan, observability.AttrMCPMethodName); ok { + t.Error("mcp.method.name must not be set on a non-MCP (function) tool") + } +} + +// TestExecuteMCPToolSpanUsesExtensionType asserts an MCP-namespaced tool +// ("__") is typed "extension" and carries mcp.method.name. +func TestExecuteMCPToolSpanUsesExtensionType(t *testing.T) { + rec := genAIToolRun(t, "linear__create_issue", true) + + toolSpan, ok := rec.FindSpan("tool.linear__create_issue") + if !ok { + t.Fatal("missing tool.linear__create_issue span") + } + if got, _ := findAttr(toolSpan, observability.AttrGenAIToolType); got != observability.ToolTypeExtension { + t.Errorf("gen_ai.tool.type = %q; want %q for an MCP tool", got, observability.ToolTypeExtension) + } + if got, ok := findAttr(toolSpan, observability.AttrMCPMethodName); !ok || got != observability.MCPMethodToolsCall { + t.Errorf("mcp.method.name = %q (ok=%v); want %q", got, ok, observability.MCPMethodToolsCall) + } +} + +// TestExecuteNamespacedNonMCPToolNotTypedAsExtension guards the fix for the +// name-heuristic edge: a NON-MCP namespaced tool (API per-op "__") +// shares the "__" shape but must be typed "function" with no mcp.method.name, +// because classification is driven by the registry's MCPSource marker. +func TestExecuteNamespacedNonMCPToolNotTypedAsExtension(t *testing.T) { + rec := genAIToolRun(t, "weatherapi__forecast", false) + + toolSpan, ok := rec.FindSpan("tool.weatherapi__forecast") + if !ok { + t.Fatal("missing tool.weatherapi__forecast span") + } + if got, _ := findAttr(toolSpan, observability.AttrGenAIToolType); got != observability.ToolTypeFunction { + t.Errorf("gen_ai.tool.type = %q; want %q for a non-MCP namespaced tool", got, observability.ToolTypeFunction) + } + if _, ok := findAttr(toolSpan, observability.AttrMCPMethodName); ok { + t.Error("mcp.method.name must not be set on a non-MCP namespaced tool") + } +} diff --git a/forge-core/runtime/loop_test.go b/forge-core/runtime/loop_test.go index 1b58047e..08649476 100644 --- a/forge-core/runtime/loop_test.go +++ b/forge-core/runtime/loop_test.go @@ -30,6 +30,9 @@ func (m *mockLLMClient) ModelID() string { return "test-model" } type mockToolExecutor struct { executeFunc func(ctx context.Context, name string, arguments json.RawMessage) (string, error) toolDefs []llm.ToolDefinition + // mcpNames marks which tool names the executor should treat as + // MCP-backed (mirrors the registry's MCPSource marker). nil => none. + mcpNames map[string]bool } func (m *mockToolExecutor) Execute(ctx context.Context, name string, arguments json.RawMessage) (string, error) { @@ -40,6 +43,10 @@ func (m *mockToolExecutor) ToolDefinitions() []llm.ToolDefinition { return m.toolDefs } +func (m *mockToolExecutor) IsMCPTool(name string) bool { + return m.mcpNames[name] +} + func TestToolResultTruncation(t *testing.T) { // Generate a tool result that exceeds the proportional limit. // With CharBudget=100_000, the tool limit = 25K, so a 60K result gets truncated. diff --git a/forge-core/tools/registry.go b/forge-core/tools/registry.go index 076e46eb..3bfce232 100644 --- a/forge-core/tools/registry.go +++ b/forge-core/tools/registry.go @@ -113,6 +113,23 @@ func (r *Registry) Filter(allowed []string) *Registry { return filtered } +// IsMCPTool reports whether the named tool was discovered from an MCP +// server (implements the MCPSource marker). This is the authoritative +// signal for MCP-backedness — distinct from the "__" name +// shape, which NamespacedSource tools (e.g. per-op API tools "__") +// also use without being MCP. Returns false for unknown or non-MCP tools. +// Satisfies the engine.ToolExecutor interface. +func (r *Registry) IsMCPTool(name string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + t, ok := r.tools[name] + if !ok { + return false + } + _, isMCP := t.(MCPSource) + return isMCP +} + // ToolDefinitions returns LLM tool definitions for all registered tools. // This method satisfies the engine.ToolExecutor interface. func (r *Registry) ToolDefinitions() []llm.ToolDefinition {