Chat Agent bugfix to treat workspace file references and attachments separately

ts/packages/agents/chat/jest.config.cjs

@@ -0,0 +1,4 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+module.exports = require("../../../jest.config.js");

ts/packages/agents/chat/package.json

@@ -23,8 +23,11 @@
     "build": "npm run tsc",
     "postbuild": "copyfiles -u 1 \"src/**/config.json\" dist",
     "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log",
+    "jest-esm": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js",
     "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
+    "test": "npm run test:local",
+    "test:local": "pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"",
     "tsc": "tsc -b"
   },
   "dependencies": {
@@ -37,7 +40,9 @@
     "typechat-utils": "workspace:*"
   },
   "devDependencies": {
+    "@types/jest": "^29.5.7",
     "copyfiles": "^2.4.1",
+    "jest": "^29.7.0",
     "prettier": "^3.5.3",
     "rimraf": "^6.0.1",
     "typescript": "~5.4.5"

ts/packages/agents/chat/src/chatResponseActionSchema.ts

@@ -34,7 +34,10 @@ export interface GenerateResponseAction {
         userRequestEntities: Entity[];
         // ALL the actions and entities present in the generated text
         generatedTextEntities: Entity[];
-        // The file names of any attachments
+        // File names of images the user uploaded as attachments this turn (each
+        // is provided to you alongside its image as "File Name: ..."). Only list
+        // those uploaded attachment names here; do not add files that are only
+        // referenced through the editor or workspace context.
         relatedFiles?: string[];
     };
 }

ts/packages/agents/chat/src/chatResponseHandler.ts

@@ -1,7 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { getImageElement, getMimeType } from "typechat-utils";
+import {
+    getAttachmentFileName,
+    isImageAttachment,
+    rehydrateImageAttachments,
+} from "typechat-utils";
 import {
     ChatResponseAction,
     Entity,
@@ -28,31 +32,15 @@ export async function executeChatResponseAction(
     return handleChatResponse(chatAction, context);
 }
 
-async function rehydrateImages(context: ActionContext, files: string[]) {
-    let html = "<div>";
-
-    for (let i = 0; i < files.length; i++) {
-        let name = files[i];
-        console.log(`Rehydrating Image ${name}`);
-        if (files[i].lastIndexOf("\\") > -1) {
-            name = files[i].substring(files[i].lastIndexOf("\\") + 1);
-        }
-
-        const a = await context.sessionContext.sessionStorage?.read(
-            `\\..\\user_files\\${name}`,
-            "base64",
-        );
-
-        if (a) {
-            html += getImageElement(
-                `data:image/${getMimeType(name.substring(name.indexOf(".")))};base64,${a}`,
-            );
-        }
-    }
-
-    html += "</div>";
-
-    return html;
+// Build the file entity the dispatcher records for a related file. Uploaded
+// image attachments are tagged as images; anything else (e.g. a workspace file
+// the user referenced from the editor) is recorded as a plain file reference.
+export function relatedFileToEntity(file: string): Entity {
+    const name = getAttachmentFileName(file);
+    return {
+        name,
+        type: isImageAttachment(name) ? ["file", "image", "data"] : ["file"],
+    };
 }
 
 async function handleChatResponse(
@@ -67,7 +55,7 @@ async function handleChatResponse(
 
         case "showImageFile":
             return createActionResultFromHtmlDisplay(
-                `<div>${await rehydrateImages(context, chatAction.parameters.files)}</div>`,
+                `<div>${await rehydrateImageAttachments(context.sessionContext.sessionStorage, chatAction.parameters.files)}</div>`,
             );
 
         default:
@@ -110,7 +98,7 @@ async function generateResponse(
             context.actionIO.appendDisplay(
                 {
                     type: "html",
-                    content: `<div class='chat-smallImage'>${await rehydrateImages(context, generateResponseAction.parameters.relatedFiles!)}</div>`,
+                    content: `<div class='chat-smallImage'>${await rehydrateImageAttachments(context.sessionContext.sessionStorage, generateResponseAction.parameters.relatedFiles!)}</div>`,
                 },
                 "block",
             );
@@ -124,17 +112,10 @@ async function generateResponse(
         }
 
         if (generateResponseAction.parameters.relatedFiles !== undefined) {
-            const fileEntities: Entity[] = new Array<Entity>();
-            for (const file of generateResponseAction.parameters.relatedFiles) {
-                let name = file;
-                if (file.lastIndexOf("\\") > -1) {
-                    name = file.substring(file.lastIndexOf("\\") + 1);
-                }
-                fileEntities.push({
-                    name,
-                    type: ["file", "image", "data"],
-                });
-            }
+            const fileEntities: Entity[] =
+                generateResponseAction.parameters.relatedFiles.map(
+                    relatedFileToEntity,
+                );
 
             logEntities("File Entities:", fileEntities);
             result.entities = result.entities.concat(fileEntities);

ts/packages/agents/chat/test/chatResponseHandler.spec.ts

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { ActionContext, TypeAgentAction } from "@typeagent/agent-sdk";
+import { ChatResponseAction } from "../src/chatResponseActionSchema.js";
+import {
+    executeChatResponseAction,
+    relatedFileToEntity,
+} from "../src/chatResponseHandler.js";
+
+describe("relatedFileToEntity", () => {
+    it("tags an uploaded image attachment as an image entity", () => {
+        const entity = relatedFileToEntity("attachment_0.png");
+
+        expect(entity.name).toBe("attachment_0.png");
+        expect(entity.type).toEqual(["file", "image", "data"]);
+    });
+
+    it("records a highlighted workspace file as a plain file reference", () => {
+        const entity = relatedFileToEntity(
+            "pipelines/azure-build-docker-container.yml",
+        );
+
+        // Base name only, and not tagged as an image.
+        expect(entity.name).toBe("azure-build-docker-container.yml");
+        expect(entity.type).toEqual(["file"]);
+    });
+});
+
+describe("executeChatResponseAction", () => {
+    it("answers with a highlighted workspace file without reading storage", async () => {
+        const reads: string[] = [];
+        const context = {
+            actionIO: {
+                setDisplay: () => {},
+                appendDisplay: () => {},
+            },
+            sessionContext: {
+                sessionStorage: {
+                    read: (path: string) => {
+                        reads.push(path);
+                        return Promise.reject(new Error("ENOENT"));
+                    },
+                },
+            },
+        } as unknown as ActionContext;
+
+        const action = {
+            actionName: "generateResponse",
+            parameters: {
+                originalRequest: "what does this pipeline do",
+                generatedText: "It builds and publishes a container image.",
+                userRequestEntities: [],
+                generatedTextEntities: [],
+                relatedFiles: ["pipelines/azure-build-docker-container.yml"],
+            },
+        } as unknown as TypeAgentAction<ChatResponseAction>;
+
+        const result = await executeChatResponseAction(action, context);
+
+        // A non-image editor reference must neither hit storage nor throw, and
+        // is recorded as a plain file entity.
+        expect(reads).toEqual([]);
+        expect(result?.entities).toContainEqual({
+            name: "azure-build-docker-container.yml",
+            type: ["file"],
+        });
+    });
+});

ts/packages/agents/chat/test/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": ".",
+    "outDir": "../dist/test",
+    "types": ["node", "jest"]
+  },
+  "include": ["./**/*"],
+  "ts-node": {
+    "esm": true
+  },
+  "references": [{ "path": "../src" }]
+}

ts/packages/agents/chat/tsconfig.json

@@ -4,7 +4,7 @@
     "composite": true
   },
   "include": [],
-  "references": [{ "path": "./src" }],
+  "references": [{ "path": "./src" }, { "path": "./test" }],
   "ts-node": {
     "esm": true
   }

ts/packages/dispatcher/dispatcher/src/search/search.ts

@@ -15,7 +15,7 @@ import { ActionContext, ActionResult, Entity } from "@typeagent/agent-sdk";
 import { conversation } from "knowledge-processor";
 //import { getLookupSettings, handleLookup } from "./internet.js";
 import registerDebug from "debug";
-import { getImageElement, getMimeType } from "typechat-utils";
+import { rehydrateImageAttachments } from "typechat-utils";
 import { lookupAndAnswerFromMemory } from "../context/memory.js";
 
 const debug = registerDebug("typeagent:dispatcher:lookup");
@@ -77,7 +77,7 @@ async function getAnswerFromConversationManager(
             );
 
             return createActionResultFromHtmlDisplay(
-                `<div>${matches.response.answer.answer!}</div><div class='chat-smallImage'>${await rehydrateImages(context, imageNames)}</div>`,
+                `<div>${matches.response.answer.answer!}</div><div class='chat-smallImage'>${await rehydrateImageAttachments(context.sessionContext.sessionStorage, imageNames)}</div>`,
                 matches.response.answer.answer!,
                 matchEntities,
             );
@@ -161,38 +161,3 @@ function compositeEntityToEntity(entity: conversation.CompositeEntity): Entity {
         type: [...entity.type, conversation.KnownEntityTypes.Memorized],
     };
 }
-
-async function rehydrateImages(
-    context: ActionContext<CommandHandlerContext>,
-    files: (string | undefined)[],
-) {
-    let html = "<div>";
-
-    if (files) {
-        for (let i = 0; i < files.length; i++) {
-            let name = files[i];
-
-            if (files[i] && name) {
-                console.log(`Rehydrating Image ${name}`);
-                if (files[i]!.lastIndexOf("\\") > -1) {
-                    name = files[i]!.substring(files[i]!.lastIndexOf("\\") + 1);
-                }
-
-                const a = await context.sessionContext.sessionStorage?.read(
-                    `\\..\\user_files\\${name}`,
-                    "base64",
-                );
-
-                if (a) {
-                    html += getImageElement(
-                        `data:image/${getMimeType(name.substring(name.indexOf(".")))};base64,${a}`,
-                    );
-                }
-            }
-        }
-    }
-
-    html += "</div>";
-
-    return html;
-}

ts/packages/utils/typechatUtils/src/image.ts

@@ -17,9 +17,13 @@ import {
     ReverseGeocodeAddressLookup,
 } from "./location.js";
 import { openai } from "@typeagent/aiclient";
+import { getMimeTypeFromFileExtension } from "./mimeTypes.js";
 import fs from "node:fs";
 import path from "node:path";
 import { parse } from "date-fns";
+import registerDebug from "debug";
+
+const debug = registerDebug("typeagent:typechat-utils:image");
 
 export class CachedImageWithDetails {
     constructor(
@@ -39,6 +43,72 @@ export function getImageElement(imgData: string): string {
     return `<img class="chat-input-image" src="${imgData}" />`;
 }
 
+// Strip an attachment path down to its bare file name. The path can arrive with
+// Windows (\) or POSIX (/) separators depending on its origin (an uploaded
+// attachment vs. a highlighted editor file), so cut at whichever comes last.
+export function getAttachmentFileName(file: string): string {
+    const sep = Math.max(file.lastIndexOf("\\"), file.lastIndexOf("/"));
+    return sep > -1 ? file.substring(sep + 1) : file;
+}
+
+// The image MIME type for a file name, or undefined when the extension is not a
+// supported image type (or has no extension). Used to tell an embeddable image
+// attachment apart from a non-image reference without touching storage.
+function imageMimeType(fileName: string): string | undefined {
+    const ext = fileName.substring(fileName.lastIndexOf("."));
+    try {
+        const mime = getMimeTypeFromFileExtension(ext);
+        return mime.startsWith("image/") ? mime : undefined;
+    } catch {
+        return undefined;
+    }
+}
+
+export function isImageAttachment(fileName: string): boolean {
+    return imageMimeType(fileName) !== undefined;
+}
+
+// Build an HTML block of inline <img> data URLs for uploaded image attachments.
+// Each name in `files` is read from the session's user_files staging directory
+// (one level up from the calling agent's storage). Non-image references (e.g. a
+// highlighted .yml from editor context) are skipped with no read, and an image
+// name missing from user_files is skipped when the read fails.
+export async function rehydrateImageAttachments(
+    storage: Storage | undefined,
+    files: (string | undefined)[],
+): Promise<string> {
+    let html = "<div>";
+
+    for (const file of files) {
+        if (!file) {
+            continue;
+        }
+        const name = getAttachmentFileName(file);
+        const mimeType = imageMimeType(name);
+        if (mimeType === undefined) {
+            continue;
+        }
+
+        try {
+            // Build the path with path.join so the separators match the host;
+            // a literal "\\..\\" only resolves correctly on Windows.
+            const data = await storage?.read(
+                path.join("..", "user_files", name),
+                "base64",
+            );
+            if (data) {
+                html += getImageElement(`data:${mimeType};base64,${data}`);
+            }
+        } catch (e) {
+            debug(`skipping image attachment ${name}: ${e}`);
+        }
+    }
+
+    html += "</div>";
+
+    return html;
+}
+
 export function extractRelevantExifTags(exifTags: ExifReader.Tags | undefined) {
     let tags: string = "";