From 6fe45e03a13b04d51e99c02d90d6ef6079ce2a13 Mon Sep 17 00:00:00 2001 From: sbrooke Date: Fri, 17 Jul 2026 16:24:15 -0400 Subject: [PATCH] fix(core): hide caller data from the model, fill by key at the sink (TAB-1104) Caller data was JSON-rendered into the task prompt, so the model could read a secret and place it into a goto/tabstack URL or done(). Render only the data KEYS; add fill_user_data(ref, key) which substitutes the value at the browser sink and never returns or emits it (only a {{key}} mask). It runs the same assessFill gate as regular fill and records agentFilledRefs (never approvedRefs), so it cannot fill caller data into fields on untrusted hosts. The model never holds the value, so it cannot exfiltrate it regardless of domain or encoding. Keys-only prompt, hasUserData threaded into the tool list and step-error prompt, select/checkbox mapping. --- packages/core/src/prompts.ts | 29 +++- packages/core/src/tools/webActionTools.ts | 116 +++++++++++++- packages/core/src/webAgent.ts | 14 +- packages/core/test/prompts.test.ts | 113 +++----------- .../core/test/tools/webActionTools.test.ts | 144 ++++++++++++++++++ 5 files changed, 318 insertions(+), 98 deletions(-) diff --git a/packages/core/src/prompts.ts b/packages/core/src/prompts.ts index 6e2a2b45..bbe424f6 100644 --- a/packages/core/src/prompts.ts +++ b/packages/core/src/prompts.ts @@ -90,6 +90,12 @@ export const TOOL_STRINGS = { formDescription: "Brief description of the form and its purpose (e.g., 'Shipping address form', 'Account registration')", }, + fillUserData: { + description: + "Fill a field with one of the caller's Input Data values, by key. The value is inserted directly into the field and is NEVER shown to you. Use this for any field that should receive a caller-provided data value (personal info, credentials, etc.).", + ref: "Element reference from page snapshot (e.g., E###)", + key: "The Input Data key whose value to insert (see the Available keys list)", + }, webSearch: { description: "Search the web for information. Returns the search results page as markdown. Use when you need to find websites or information but don't know the URL.", @@ -176,6 +182,7 @@ function buildToolExamples( hasInteractive: boolean = false, hasFileUpload: boolean = false, advertisedUploadFiles: string[] = [], + hasUserData: boolean = false, ): string { const lines = [ `- click({"ref": "${TOOL_STRINGS.webActions.common.elementRefExample}"}) - ${TOOL_STRINGS.webActions.click.description}`, @@ -223,6 +230,12 @@ function buildToolExamples( ); } + if (hasUserData) { + lines.push( + `- fill_user_data({"ref": "E###", "key": "membership_code"}) - ${TOOL_STRINGS.webActions.fillUserData.description}`, + ); + } + lines.push( `- done({"result": "your final answer"}) - ${TOOL_STRINGS.webActions.done.description}`, `- abort({"reason": "what was tried and why it failed"}) - ${TOOL_STRINGS.webActions.abort.description}`, @@ -452,6 +465,7 @@ const buildActionLoopSystemPrompt = ( hasInteractive: boolean = false, hasFileUpload: boolean = false, advertisedUploadFiles: string[] = [], + hasUserData: boolean = false, ) => actionLoopSystemPromptTemplate({ hasGuardrails, @@ -465,6 +479,7 @@ const buildActionLoopSystemPrompt = ( hasInteractive, hasFileUpload, advertisedUploadFiles, + hasUserData, ), currentDate: getCurrentFormattedDate(), }); @@ -481,11 +496,9 @@ Today's Date: {{ currentDate }} Task: {{ task }} Success Criteria: {{ successCriteria }} Plan: {{ plan }} -{% if data %} -Input Data: -\`\`\`json -{{ data }} -\`\`\` +{% if dataKeys %} +Input Data available (values hidden for your security — you cannot see them). Available keys: {{ dataKeys }} +To put one of these values into a field, call fill_user_data(ref, key). Never ask for or guess the values. {% endif %} {% if guardrails %} @@ -501,7 +514,7 @@ export const buildTaskAndPlanPrompt = ( task: string, successCriteria: string, plan: string, - data?: any, + dataKeys?: string[], guardrails?: string | null, ) => taskAndPlanTemplate({ @@ -509,7 +522,7 @@ export const buildTaskAndPlanPrompt = ( successCriteria, plan, currentDate: getCurrentFormattedDate(), - data: data ? JSON.stringify(data, null, 2) : null, + dataKeys: dataKeys && dataKeys.length ? dataKeys.join(", ") : null, guardrails, }); @@ -594,6 +607,7 @@ export const buildStepErrorFeedbackPrompt = ( hasTabstack: boolean = false, hasFileUpload: boolean = false, advertisedUploadFiles: string[] = [], + hasUserData: boolean = false, ) => stepErrorFeedbackTemplate({ error, @@ -604,6 +618,7 @@ export const buildStepErrorFeedbackPrompt = ( false, hasFileUpload, advertisedUploadFiles, + hasUserData, ), }); diff --git a/packages/core/src/tools/webActionTools.ts b/packages/core/src/tools/webActionTools.ts index 152e0cef..fb778b0c 100644 --- a/packages/core/src/tools/webActionTools.ts +++ b/packages/core/src/tools/webActionTools.ts @@ -9,6 +9,7 @@ import { tool, type ToolSet } from "ai"; import { z } from "zod"; import { AriaBrowser, + FieldMetadata, FileUploadConfig, PageAction, SCROLL_DIRECTIONS, @@ -46,6 +47,12 @@ interface WebActionContext { allowFileUpload?: false | FileUploadConfig; /** Timeout for LLM provider calls in milliseconds (used by extract). */ llmProviderTimeoutMs?: number; + /** + * Caller-supplied Input Data, keyed by name. When present with ≥1 key, the + * fill_user_data tool is exposed so the agent can fill fields by key without + * ever seeing the values — they are substituted directly at the browser sink. + */ + data?: Record; } /** @@ -297,6 +304,18 @@ async function performActionWithValidation( ); } +/** + * Choose the browser action for filling a caller-data field based on its type: + * field", async () => { + mockBrowser.url = "https://example.com"; + mockBrowser.fieldMetadata.set("E9", { + ref: "E9", + tagName: "select", + inputType: null, + role: "combobox", + name: "country", + label: "Country", + placeholder: null, + autocomplete: null, + isContentEditable: false, + formId: "signup", + formAction: "https://example.com/signup", + formMethod: "post", + }); + context.firewall = { trustedHostnames: new Set(["example.com"]), unsafeMode: false }; + udTools = createWebActionTools(context); + const performActionSpy = vi.spyOn(mockBrowser, "performAction"); + + await udTools.fill_user_data.execute({ ref: "E9", key: "membership_code" }); + + expect(performActionSpy).toHaveBeenCalledWith("E9", PageAction.Select, SECRET); + }); + + it("returns a recoverable failure when performAction throws BrowserException", async () => { + mockBrowser.url = "https://example.com"; + context.firewall = { trustedHostnames: new Set(["example.com"]), unsafeMode: false }; + udTools = createWebActionTools(context); + vi.spyOn(mockBrowser, "performAction").mockRejectedValueOnce( + new BrowserActionException("fill", "element is not editable"), + ); + + const result = await udTools.fill_user_data.execute({ ref: "E12", key: "membership_code" }); + + expect(result).toEqual({ + success: false, + action: "fill_user_data", + ref: "E12", + error: "element is not editable", + isRecoverable: true, + }); + expect(context.agentFilledRefs.has("E12")).toBe(false); + expect(JSON.stringify(result)).not.toContain(SECRET); + }); + }); });