diff --git a/packages/core/src/prompts.ts b/packages/core/src/prompts.ts index 6e2a2b45..bbe424f6 100644 --- a/packages/core/src/prompts.ts +++ b/packages/core/src/prompts.ts @@ -90,6 +90,12 @@ export const TOOL_STRINGS = { formDescription: "Brief description of the form and its purpose (e.g., 'Shipping address form', 'Account registration')", }, + fillUserData: { + description: + "Fill a field with one of the caller's Input Data values, by key. The value is inserted directly into the field and is NEVER shown to you. Use this for any field that should receive a caller-provided data value (personal info, credentials, etc.).", + ref: "Element reference from page snapshot (e.g., E###)", + key: "The Input Data key whose value to insert (see the Available keys list)", + }, webSearch: { description: "Search the web for information. Returns the search results page as markdown. Use when you need to find websites or information but don't know the URL.", @@ -176,6 +182,7 @@ function buildToolExamples( hasInteractive: boolean = false, hasFileUpload: boolean = false, advertisedUploadFiles: string[] = [], + hasUserData: boolean = false, ): string { const lines = [ `- click({"ref": "${TOOL_STRINGS.webActions.common.elementRefExample}"}) - ${TOOL_STRINGS.webActions.click.description}`, @@ -223,6 +230,12 @@ function buildToolExamples( ); } + if (hasUserData) { + lines.push( + `- fill_user_data({"ref": "E###", "key": "membership_code"}) - ${TOOL_STRINGS.webActions.fillUserData.description}`, + ); + } + lines.push( `- done({"result": "your final answer"}) - ${TOOL_STRINGS.webActions.done.description}`, `- abort({"reason": "what was tried and why it failed"}) - ${TOOL_STRINGS.webActions.abort.description}`, @@ -452,6 +465,7 @@ const buildActionLoopSystemPrompt = ( hasInteractive: boolean = false, hasFileUpload: boolean = false, advertisedUploadFiles: string[] = [], + hasUserData: boolean = false, ) => actionLoopSystemPromptTemplate({ hasGuardrails, @@ -465,6 +479,7 @@ const buildActionLoopSystemPrompt = ( hasInteractive, hasFileUpload, advertisedUploadFiles, + hasUserData, ), currentDate: getCurrentFormattedDate(), }); @@ -481,11 +496,9 @@ Today's Date: {{ currentDate }} Task: {{ task }} Success Criteria: {{ successCriteria }} Plan: {{ plan }} -{% if data %} -Input Data: -\`\`\`json -{{ data }} -\`\`\` +{% if dataKeys %} +Input Data available (values hidden for your security — you cannot see them). Available keys: {{ dataKeys }} +To put one of these values into a field, call fill_user_data(ref, key). Never ask for or guess the values. {% endif %} {% if guardrails %} @@ -501,7 +514,7 @@ export const buildTaskAndPlanPrompt = ( task: string, successCriteria: string, plan: string, - data?: any, + dataKeys?: string[], guardrails?: string | null, ) => taskAndPlanTemplate({ @@ -509,7 +522,7 @@ export const buildTaskAndPlanPrompt = ( successCriteria, plan, currentDate: getCurrentFormattedDate(), - data: data ? JSON.stringify(data, null, 2) : null, + dataKeys: dataKeys && dataKeys.length ? dataKeys.join(", ") : null, guardrails, }); @@ -594,6 +607,7 @@ export const buildStepErrorFeedbackPrompt = ( hasTabstack: boolean = false, hasFileUpload: boolean = false, advertisedUploadFiles: string[] = [], + hasUserData: boolean = false, ) => stepErrorFeedbackTemplate({ error, @@ -604,6 +618,7 @@ export const buildStepErrorFeedbackPrompt = ( false, hasFileUpload, advertisedUploadFiles, + hasUserData, ), }); diff --git a/packages/core/src/tools/webActionTools.ts b/packages/core/src/tools/webActionTools.ts index eb728539..f8c6ff51 100644 --- a/packages/core/src/tools/webActionTools.ts +++ b/packages/core/src/tools/webActionTools.ts @@ -9,6 +9,7 @@ import { tool, type ToolSet } from "ai"; import { z } from "zod"; import { AriaBrowser, + FieldMetadata, FileUploadConfig, PageAction, SCROLL_DIRECTIONS, @@ -48,6 +49,12 @@ interface WebActionContext { allowFileUpload?: false | FileUploadConfig; /** Timeout for LLM provider calls in milliseconds (used by extract). */ llmProviderTimeoutMs?: number; + /** + * Caller-supplied Input Data, keyed by name. When present with ≥1 key, the + * fill_user_data tool is exposed so the agent can fill fields by key without + * ever seeing the values — they are substituted directly at the browser sink. + */ + data?: Record; } /** @@ -275,6 +282,18 @@ async function performActionWithValidation( ); } +/** + * Choose the browser action for filling a caller-data field based on its type: + * field", async () => { + mockBrowser.url = "https://example.com"; + mockBrowser.fieldMetadata.set("E9", { + ref: "E9", + tagName: "select", + inputType: null, + role: "combobox", + name: "country", + label: "Country", + placeholder: null, + autocomplete: null, + isContentEditable: false, + formId: "signup", + formAction: "https://example.com/signup", + formMethod: "post", + }); + context.firewall = { trustedHostnames: new Set(["example.com"]), unsafeMode: false }; + udTools = createWebActionTools(context); + const performActionSpy = vi.spyOn(mockBrowser, "performAction"); + + await udTools.fill_user_data.execute({ ref: "E9", key: "membership_code" }); + + expect(performActionSpy).toHaveBeenCalledWith("E9", PageAction.Select, SECRET); + }); + + it("returns a recoverable failure when performAction throws BrowserException", async () => { + mockBrowser.url = "https://example.com"; + context.firewall = { trustedHostnames: new Set(["example.com"]), unsafeMode: false }; + udTools = createWebActionTools(context); + vi.spyOn(mockBrowser, "performAction").mockRejectedValueOnce( + new BrowserActionException("fill", "element is not editable"), + ); + + const result = await udTools.fill_user_data.execute({ ref: "E12", key: "membership_code" }); + + expect(result).toEqual({ + success: false, + action: "fill_user_data", + ref: "E12", + error: "element is not editable", + isRecoverable: true, + }); + expect(context.agentFilledRefs.has("E12")).toBe(false); + expect(JSON.stringify(result)).not.toContain(SECRET); + }); + }); });