diff --git a/ts/packages/agents/chat/jest.config.cjs b/ts/packages/agents/chat/jest.config.cjs new file mode 100644 index 0000000000..f475768a12 --- /dev/null +++ b/ts/packages/agents/chat/jest.config.cjs @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +module.exports = require("../../../jest.config.js"); diff --git a/ts/packages/agents/chat/package.json b/ts/packages/agents/chat/package.json index d283edd890..331c53952f 100644 --- a/ts/packages/agents/chat/package.json +++ b/ts/packages/agents/chat/package.json @@ -23,8 +23,11 @@ "build": "npm run tsc", "postbuild": "copyfiles -u 1 \"src/**/config.json\" dist", "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log", + "jest-esm": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js", "prettier": "prettier --check . --ignore-path ../../../.prettierignore", "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore", + "test": "npm run test:local", + "test:local": "pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"", "tsc": "tsc -b" }, "dependencies": { @@ -37,7 +40,9 @@ "typechat-utils": "workspace:*" }, "devDependencies": { + "@types/jest": "^29.5.7", "copyfiles": "^2.4.1", + "jest": "^29.7.0", "prettier": "^3.5.3", "rimraf": "^6.0.1", "typescript": "~5.4.5" diff --git a/ts/packages/agents/chat/src/chatResponseActionSchema.ts b/ts/packages/agents/chat/src/chatResponseActionSchema.ts index 29e2c4818b..5dd7cbf6d3 100644 --- a/ts/packages/agents/chat/src/chatResponseActionSchema.ts +++ b/ts/packages/agents/chat/src/chatResponseActionSchema.ts @@ -34,7 +34,10 @@ export interface GenerateResponseAction { userRequestEntities: Entity[]; // ALL the actions and entities present in the generated text generatedTextEntities: Entity[]; - // The file names of any attachments + // File names of images the user uploaded as attachments this turn (each + // is provided to you alongside its image as "File Name: ..."). Only list + // those uploaded attachment names here; do not add files that are only + // referenced through the editor or workspace context. relatedFiles?: string[]; }; } diff --git a/ts/packages/agents/chat/src/chatResponseHandler.ts b/ts/packages/agents/chat/src/chatResponseHandler.ts index 010bb0f13c..99f970472c 100644 --- a/ts/packages/agents/chat/src/chatResponseHandler.ts +++ b/ts/packages/agents/chat/src/chatResponseHandler.ts @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { getImageElement, getMimeType } from "typechat-utils"; +import { + getAttachmentFileName, + isImageAttachment, + rehydrateImageAttachments, +} from "typechat-utils"; import { ChatResponseAction, Entity, @@ -28,31 +32,15 @@ export async function executeChatResponseAction( return handleChatResponse(chatAction, context); } -async function rehydrateImages(context: ActionContext, files: string[]) { - let html = "
"; - - for (let i = 0; i < files.length; i++) { - let name = files[i]; - console.log(`Rehydrating Image ${name}`); - if (files[i].lastIndexOf("\\") > -1) { - name = files[i].substring(files[i].lastIndexOf("\\") + 1); - } - - const a = await context.sessionContext.sessionStorage?.read( - `\\..\\user_files\\${name}`, - "base64", - ); - - if (a) { - html += getImageElement( - `data:image/${getMimeType(name.substring(name.indexOf(".")))};base64,${a}`, - ); - } - } - - html += "
"; - - return html; +// Build the file entity the dispatcher records for a related file. Uploaded +// image attachments are tagged as images; anything else (e.g. a workspace file +// the user referenced from the editor) is recorded as a plain file reference. +export function relatedFileToEntity(file: string): Entity { + const name = getAttachmentFileName(file); + return { + name, + type: isImageAttachment(name) ? ["file", "image", "data"] : ["file"], + }; } async function handleChatResponse( @@ -67,7 +55,7 @@ async function handleChatResponse( case "showImageFile": return createActionResultFromHtmlDisplay( - `
${await rehydrateImages(context, chatAction.parameters.files)}
`, + `
${await rehydrateImageAttachments(context.sessionContext.sessionStorage, chatAction.parameters.files)}
`, ); default: @@ -110,7 +98,7 @@ async function generateResponse( context.actionIO.appendDisplay( { type: "html", - content: `
${await rehydrateImages(context, generateResponseAction.parameters.relatedFiles!)}
`, + content: `
${await rehydrateImageAttachments(context.sessionContext.sessionStorage, generateResponseAction.parameters.relatedFiles!)}
`, }, "block", ); @@ -124,17 +112,10 @@ async function generateResponse( } if (generateResponseAction.parameters.relatedFiles !== undefined) { - const fileEntities: Entity[] = new Array(); - for (const file of generateResponseAction.parameters.relatedFiles) { - let name = file; - if (file.lastIndexOf("\\") > -1) { - name = file.substring(file.lastIndexOf("\\") + 1); - } - fileEntities.push({ - name, - type: ["file", "image", "data"], - }); - } + const fileEntities: Entity[] = + generateResponseAction.parameters.relatedFiles.map( + relatedFileToEntity, + ); logEntities("File Entities:", fileEntities); result.entities = result.entities.concat(fileEntities); diff --git a/ts/packages/agents/chat/test/chatResponseHandler.spec.ts b/ts/packages/agents/chat/test/chatResponseHandler.spec.ts new file mode 100644 index 0000000000..40dbdb35fe --- /dev/null +++ b/ts/packages/agents/chat/test/chatResponseHandler.spec.ts @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ActionContext, TypeAgentAction } from "@typeagent/agent-sdk"; +import { ChatResponseAction } from "../src/chatResponseActionSchema.js"; +import { + executeChatResponseAction, + relatedFileToEntity, +} from "../src/chatResponseHandler.js"; + +describe("relatedFileToEntity", () => { + it("tags an uploaded image attachment as an image entity", () => { + const entity = relatedFileToEntity("attachment_0.png"); + + expect(entity.name).toBe("attachment_0.png"); + expect(entity.type).toEqual(["file", "image", "data"]); + }); + + it("records a highlighted workspace file as a plain file reference", () => { + const entity = relatedFileToEntity( + "pipelines/azure-build-docker-container.yml", + ); + + // Base name only, and not tagged as an image. + expect(entity.name).toBe("azure-build-docker-container.yml"); + expect(entity.type).toEqual(["file"]); + }); +}); + +describe("executeChatResponseAction", () => { + it("answers with a highlighted workspace file without reading storage", async () => { + const reads: string[] = []; + const context = { + actionIO: { + setDisplay: () => {}, + appendDisplay: () => {}, + }, + sessionContext: { + sessionStorage: { + read: (path: string) => { + reads.push(path); + return Promise.reject(new Error("ENOENT")); + }, + }, + }, + } as unknown as ActionContext; + + const action = { + actionName: "generateResponse", + parameters: { + originalRequest: "what does this pipeline do", + generatedText: "It builds and publishes a container image.", + userRequestEntities: [], + generatedTextEntities: [], + relatedFiles: ["pipelines/azure-build-docker-container.yml"], + }, + } as unknown as TypeAgentAction; + + const result = await executeChatResponseAction(action, context); + + // A non-image editor reference must neither hit storage nor throw, and + // is recorded as a plain file entity. + expect(reads).toEqual([]); + expect(result?.entities).toContainEqual({ + name: "azure-build-docker-container.yml", + type: ["file"], + }); + }); +}); diff --git a/ts/packages/agents/chat/test/tsconfig.json b/ts/packages/agents/chat/test/tsconfig.json new file mode 100644 index 0000000000..895a6f0d21 --- /dev/null +++ b/ts/packages/agents/chat/test/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "rootDir": ".", + "outDir": "../dist/test", + "types": ["node", "jest"] + }, + "include": ["./**/*"], + "ts-node": { + "esm": true + }, + "references": [{ "path": "../src" }] +} diff --git a/ts/packages/agents/chat/tsconfig.json b/ts/packages/agents/chat/tsconfig.json index acb9cb4a91..94dfc60bb1 100644 --- a/ts/packages/agents/chat/tsconfig.json +++ b/ts/packages/agents/chat/tsconfig.json @@ -4,7 +4,7 @@ "composite": true }, "include": [], - "references": [{ "path": "./src" }], + "references": [{ "path": "./src" }, { "path": "./test" }], "ts-node": { "esm": true } diff --git a/ts/packages/dispatcher/dispatcher/src/search/search.ts b/ts/packages/dispatcher/dispatcher/src/search/search.ts index 7211223ee9..59f756be86 100644 --- a/ts/packages/dispatcher/dispatcher/src/search/search.ts +++ b/ts/packages/dispatcher/dispatcher/src/search/search.ts @@ -15,7 +15,7 @@ import { ActionContext, ActionResult, Entity } from "@typeagent/agent-sdk"; import { conversation } from "knowledge-processor"; //import { getLookupSettings, handleLookup } from "./internet.js"; import registerDebug from "debug"; -import { getImageElement, getMimeType } from "typechat-utils"; +import { rehydrateImageAttachments } from "typechat-utils"; import { lookupAndAnswerFromMemory } from "../context/memory.js"; const debug = registerDebug("typeagent:dispatcher:lookup"); @@ -77,7 +77,7 @@ async function getAnswerFromConversationManager( ); return createActionResultFromHtmlDisplay( - `
${matches.response.answer.answer!}
${await rehydrateImages(context, imageNames)}
`, + `
${matches.response.answer.answer!}
${await rehydrateImageAttachments(context.sessionContext.sessionStorage, imageNames)}
`, matches.response.answer.answer!, matchEntities, ); @@ -161,38 +161,3 @@ function compositeEntityToEntity(entity: conversation.CompositeEntity): Entity { type: [...entity.type, conversation.KnownEntityTypes.Memorized], }; } - -async function rehydrateImages( - context: ActionContext, - files: (string | undefined)[], -) { - let html = "
"; - - if (files) { - for (let i = 0; i < files.length; i++) { - let name = files[i]; - - if (files[i] && name) { - console.log(`Rehydrating Image ${name}`); - if (files[i]!.lastIndexOf("\\") > -1) { - name = files[i]!.substring(files[i]!.lastIndexOf("\\") + 1); - } - - const a = await context.sessionContext.sessionStorage?.read( - `\\..\\user_files\\${name}`, - "base64", - ); - - if (a) { - html += getImageElement( - `data:image/${getMimeType(name.substring(name.indexOf(".")))};base64,${a}`, - ); - } - } - } - } - - html += "
"; - - return html; -} diff --git a/ts/packages/utils/typechatUtils/src/image.ts b/ts/packages/utils/typechatUtils/src/image.ts index 4305d2c21e..08965f670c 100644 --- a/ts/packages/utils/typechatUtils/src/image.ts +++ b/ts/packages/utils/typechatUtils/src/image.ts @@ -17,9 +17,13 @@ import { ReverseGeocodeAddressLookup, } from "./location.js"; import { openai } from "@typeagent/aiclient"; +import { getMimeTypeFromFileExtension } from "./mimeTypes.js"; import fs from "node:fs"; import path from "node:path"; import { parse } from "date-fns"; +import registerDebug from "debug"; + +const debug = registerDebug("typeagent:typechat-utils:image"); export class CachedImageWithDetails { constructor( @@ -39,6 +43,72 @@ export function getImageElement(imgData: string): string { return ``; } +// Strip an attachment path down to its bare file name. The path can arrive with +// Windows (\) or POSIX (/) separators depending on its origin (an uploaded +// attachment vs. a highlighted editor file), so cut at whichever comes last. +export function getAttachmentFileName(file: string): string { + const sep = Math.max(file.lastIndexOf("\\"), file.lastIndexOf("/")); + return sep > -1 ? file.substring(sep + 1) : file; +} + +// The image MIME type for a file name, or undefined when the extension is not a +// supported image type (or has no extension). Used to tell an embeddable image +// attachment apart from a non-image reference without touching storage. +function imageMimeType(fileName: string): string | undefined { + const ext = fileName.substring(fileName.lastIndexOf(".")); + try { + const mime = getMimeTypeFromFileExtension(ext); + return mime.startsWith("image/") ? mime : undefined; + } catch { + return undefined; + } +} + +export function isImageAttachment(fileName: string): boolean { + return imageMimeType(fileName) !== undefined; +} + +// Build an HTML block of inline data URLs for uploaded image attachments. +// Each name in `files` is read from the session's user_files staging directory +// (one level up from the calling agent's storage). Non-image references (e.g. a +// highlighted .yml from editor context) are skipped with no read, and an image +// name missing from user_files is skipped when the read fails. +export async function rehydrateImageAttachments( + storage: Storage | undefined, + files: (string | undefined)[], +): Promise { + let html = "
"; + + for (const file of files) { + if (!file) { + continue; + } + const name = getAttachmentFileName(file); + const mimeType = imageMimeType(name); + if (mimeType === undefined) { + continue; + } + + try { + // Build the path with path.join so the separators match the host; + // a literal "\\..\\" only resolves correctly on Windows. + const data = await storage?.read( + path.join("..", "user_files", name), + "base64", + ); + if (data) { + html += getImageElement(`data:${mimeType};base64,${data}`); + } + } catch (e) { + debug(`skipping image attachment ${name}: ${e}`); + } + } + + html += "
"; + + return html; +} + export function extractRelevantExifTags(exifTags: ExifReader.Tags | undefined) { let tags: string = ""; diff --git a/ts/packages/utils/typechatUtils/test/image.spec.ts b/ts/packages/utils/typechatUtils/test/image.spec.ts new file mode 100644 index 0000000000..1a530b2358 --- /dev/null +++ b/ts/packages/utils/typechatUtils/test/image.spec.ts @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Storage } from "@typeagent/agent-sdk"; +import path from "node:path"; +import { + getAttachmentFileName, + isImageAttachment, + rehydrateImageAttachments, +} from "../src/image.js"; + +type ReadFn = (path: string) => Promise; + +// Minimal Storage whose read is driven by `read` and records the paths and +// encodings it was asked for, so tests can assert on the user_files lookup. +function createStorage(read: ReadFn): { + storage: Storage; + reads: string[]; + encodings: (string | undefined)[]; +} { + const reads: string[] = []; + const encodings: (string | undefined)[] = []; + const storage = { + read: (path: string, encoding?: string) => { + reads.push(path); + encodings.push(encoding); + return read(path); + }, + } as unknown as Storage; + return { storage, reads, encodings }; +} + +describe("getAttachmentFileName", () => { + it("strips a POSIX path to its base name", () => { + expect( + getAttachmentFileName("pipelines/azure-build-docker-container.yml"), + ).toBe("azure-build-docker-container.yml"); + }); + + it("strips a Windows path to its base name", () => { + expect(getAttachmentFileName("a\\b\\photo.png")).toBe("photo.png"); + }); + + it("uses the last separator when both kinds are present", () => { + expect(getAttachmentFileName("a/b\\c/photo.png")).toBe("photo.png"); + }); + + it("returns the input unchanged when there is no separator", () => { + expect(getAttachmentFileName("photo.png")).toBe("photo.png"); + }); +}); + +describe("isImageAttachment", () => { + it.each(["photo.png", "a/b/PHOTO.JPG", "scan.jpeg", "art.gif"])( + "treats %s as an image", + (name) => { + expect(isImageAttachment(name)).toBe(true); + }, + ); + + it.each([ + "pipelines/azure-build-docker-container.yml", + "notes.ts", + "styles.css", + "README", + ])("treats %s as a non-image", (name) => { + expect(isImageAttachment(name)).toBe(false); + }); +}); + +describe("rehydrateImageAttachments", () => { + it("embeds an uploaded image that exists in user_files", async () => { + const { storage, encodings } = createStorage(() => + Promise.resolve("AAAA"), + ); + + const html = await rehydrateImageAttachments(storage, ["cat.png"]); + + // The MIME type is emitted once (regression: no doubled "image/"). + expect(html).toContain("data:image/png;base64,AAAA"); + // The image is read as base64; dropping that would corrupt the data URL. + expect(encodings).toEqual(["base64"]); + }); + + it("skips a non-image reference without touching storage", async () => { + const { storage, reads } = createStorage(() => + Promise.reject(new Error("should not be read")), + ); + + const html = await rehydrateImageAttachments(storage, [ + "pipelines/azure-build-docker-container.yml", + ]); + + expect(html).toBe("
"); + expect(reads).toEqual([]); + }); + + it("normalizes a path to a host-correct user_files lookup", async () => { + const { storage, reads } = createStorage(() => Promise.resolve("BBBB")); + + await rehydrateImageAttachments(storage, ["shots/dog.png"]); + + expect(reads).toEqual([path.join("..", "user_files", "dog.png")]); + }); + + it("skips an image name that is missing from user_files", async () => { + const { storage } = createStorage(() => + Promise.reject(new Error("ENOENT")), + ); + + const html = await rehydrateImageAttachments(storage, ["ghost.png"]); + + expect(html).toBe("
"); + }); + + it("keeps embedding later images after skipping earlier entries", async () => { + const { storage } = createStorage((path) => + path.endsWith("dog.png") + ? Promise.resolve("CCCC") + : Promise.reject(new Error("ENOENT")), + ); + + const html = await rehydrateImageAttachments(storage, [ + undefined, + "notes.yml", + "ghost.png", + "dog.png", + ]); + + expect(html).toContain("data:image/png;base64,CCCC"); + }); +}); diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index 4d7709553c..888dc1fa63 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -2290,9 +2290,15 @@ importers: specifier: workspace:* version: link:../../utils/typechatUtils devDependencies: + '@types/jest': + specifier: ^29.5.7 + version: 29.5.14 copyfiles: specifier: ^2.4.1 version: 2.4.1 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@26.1.1)(ts-node@10.9.2(@types/node@26.1.1)(typescript@5.4.5)) prettier: specifier: ^3.5.3 version: 3.5.3 @@ -20415,7 +20421,7 @@ snapshots: fs-extra: 10.1.0 isbinaryfile: 4.0.10 minimist: 1.2.8 - plist: 3.1.0 + plist: 3.1.1 transitivePeerDependencies: - supports-color @@ -20438,7 +20444,7 @@ snapshots: dir-compare: 4.2.0 fs-extra: 11.3.6 minimatch: 9.0.9 - plist: 3.1.0 + plist: 3.1.1 transitivePeerDependencies: - supports-color @@ -25123,8 +25129,7 @@ snapshots: '@xmldom/xmldom@0.8.13': {} - '@xmldom/xmldom@0.9.10': - optional: true + '@xmldom/xmldom@0.9.10': {} '@xtuc/ieee754@1.2.0': {} @@ -29755,7 +29760,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 26.1.1 + '@types/node': 22.20.1 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -31129,7 +31134,7 @@ snapshots: node-abi@4.33.0: dependencies: - semver: 7.7.4 + semver: 7.8.5 node-addon-api@1.7.2: optional: true @@ -31140,7 +31145,7 @@ snapshots: node-api-version@0.2.1: dependencies: - semver: 7.7.4 + semver: 7.8.5 node-domexception@1.0.0: {} @@ -31169,7 +31174,7 @@ snapshots: graceful-fs: 4.2.11 nopt: 9.0.0 proc-log: 6.1.0 - semver: 7.7.4 + semver: 7.8.5 tar: 7.5.20 tinyglobby: 0.2.17 undici: 6.27.0 @@ -31708,7 +31713,6 @@ snapshots: '@xmldom/xmldom': 0.9.10 base64-js: 1.5.1 xmlbuilder: 15.1.1 - optional: true pluralize@2.0.0: {}