Add response-based AI judging for experiments

src/renderer/actions/experimentActions.test.ts

@@ -44,8 +44,13 @@ const mocks = vi.hoisted(() => ({
       }>
     >(),
     gitAddWorktree: vi.fn<(payload: unknown) => Promise<{ path: string }>>(),
-    getExperimentCandidateDiff:
-      vi.fn<(payload: unknown) => Promise<{ diff: string; headCommit: string }>>(),
+    getExperimentCandidateDiff: vi.fn<
+      (payload: unknown) => Promise<{
+        diff: string;
+        headCommit: string;
+        omittedFiles?: number;
+      }>
+    >(),
     judgeExperiment: vi.fn<
       (payload: unknown) => Promise<{
         winnerThreadId: string;
@@ -86,6 +91,7 @@ const mocks = vi.hoisted(() => ({
     dbUpsertThread: vi.fn<(thread: Thread) => Promise<void>>(),
     dbDeleteThread: vi.fn<(threadId: string) => Promise<void>>(),
     dbPersistExperimentState: vi.fn<(payload: any) => Promise<void>>(),
+    dbGetThreadRuntimeItems: vi.fn<(threadId: string) => Promise<any[]>>(),
   },
   performInitialThreadLaunch: vi.fn<(input: unknown) => Promise<void>>(),
   performWorktreeRemoval:
@@ -301,6 +307,7 @@ describe("experimentActions", () => {
     mocks.bridge.dbSetState.mockResolvedValue(undefined);
     mocks.bridge.dbUpsertThread.mockResolvedValue(undefined);
     mocks.bridge.dbDeleteThread.mockResolvedValue(undefined);
+    mocks.bridge.dbGetThreadRuntimeItems.mockReset().mockResolvedValue([]);
     mocks.bridge.dbPersistExperimentState.mockImplementation(async (payload) => {
       for (const item of payload.upsertThreads) await mocks.bridge.dbUpsertThread(item.thread);
       for (const threadId of payload.deletedThreadIds) {
@@ -344,6 +351,7 @@ describe("experimentActions", () => {
             files: result.diff ? 1 : 0,
             insertions: result.diff ? 1 : 0,
             deletions: 0,
+            ...(result.omittedFiles ? { omittedFiles: result.omittedFiles } : {}),
           };
         }),
       );
@@ -359,6 +367,7 @@ describe("experimentActions", () => {
                 files: snapshot.files,
                 insertions: snapshot.insertions,
                 deletions: snapshot.deletions,
+                ...(snapshot.omittedFiles ? { omittedFiles: snapshot.omittedFiles } : {}),
               },
             });
           }
@@ -793,7 +802,9 @@ describe("experimentActions", () => {
       ],
     }));
     useExperimentStore.getState().addExperiment(experiment());
-    const pending: Array<(result: { diff: string; headCommit: string }) => void> = [];
+    const pending: Array<
+      (result: { diff: string; headCommit: string; omittedFiles?: number }) => void
+    > = [];
     mocks.bridge.getExperimentCandidateDiff.mockReset().mockImplementation(
       () =>
         new Promise((resolve) => {
@@ -804,17 +815,25 @@ describe("experimentActions", () => {
       winnerThreadId: "thread-1",
       rationale: "Solution 1 is safer.",
     });
-    const capturedThreadIds: string[] = [];
+    const captured: Array<{ threadId: string; omittedFiles?: number }> = [];
 
     const judging = crownExperiment("experiment-1", undefined, (event) => {
-      if (event.kind === "captured") capturedThreadIds.push(event.threadId);
+      if (event.kind === "captured") {
+        captured.push({
+          threadId: event.threadId,
+          ...(event.omittedFiles ? { omittedFiles: event.omittedFiles } : {}),
+        });
+      }
     });
     await vi.waitFor(() => expect(pending).toHaveLength(2));
     pending[1]!({ diff: "second", headCommit: CANDIDATE_COMMIT });
-    pending[0]!({ diff: "first", headCommit: CANDIDATE_COMMIT });
+    pending[0]!({ diff: "first", headCommit: CANDIDATE_COMMIT, omittedFiles: 83 });
 
     await expect(judging).resolves.toBe(true);
-    expect(capturedThreadIds).toEqual(["thread-1", "thread-2"]);
+    expect(captured).toEqual([
+      { threadId: "thread-1", omittedFiles: 83 },
+      { threadId: "thread-2" },
+    ]);
   });
 
   it("uses an installed one-shot judge even when it is not a candidate provider", async () => {
@@ -894,6 +913,68 @@ describe("experimentActions", () => {
     );
   });
 
+  it("compares chat responses without capturing candidate files", async () => {
+    useAppStore.setState((state) => ({
+      ...state,
+      threads: [
+        thread("thread-1", "/repo/one", "poracode/one"),
+        thread("thread-2", "/repo/two", "poracode/two"),
+      ],
+    }));
+    useExperimentStore.getState().addExperiment(experiment());
+    mocks.bridge.dbGetThreadRuntimeItems.mockImplementation(async (threadId) => [
+      {
+        id: `${threadId}-answer`,
+        type: "assistant_message",
+        state: "completed",
+        payload: {
+          content: [
+            { kind: "text", text: threadId === "thread-1" ? "First answer" : "Second answer" },
+          ],
+        },
+        streams: {},
+      },
+    ]);
+    mocks.bridge.judgeExperimentSnapshot.mockResolvedValueOnce({
+      hash: "response-hash",
+      winnerThreadId: "thread-2",
+      rationale: "The second answer is more complete.",
+      assessments: [
+        { threadId: "thread-1", rationale: "Brief." },
+        { threadId: "thread-2", rationale: "Complete." },
+      ],
+    });
+
+    await expect(
+      crownExperiment("experiment-1", {
+        agentKind: "codex",
+        model: "gpt-5-mini",
+        effort: "low",
+        fast: false,
+        mode: "responses",
+      }),
+    ).resolves.toBe(true);
+
+    expect(mocks.bridge.judgeExperimentSnapshot).toHaveBeenCalledWith(
+      expect.objectContaining({
+        mode: "responses",
+        responses: [
+          { threadId: "thread-1", response: "Assistant:\nFirst answer" },
+          { threadId: "thread-2", response: "Assistant:\nSecond answer" },
+        ],
+      }),
+    );
+    expect(mocks.bridge.getExperimentCandidateDiff).not.toHaveBeenCalled();
+    expect(useExperimentStore.getState().experiments["experiment-1"]?.crown).toMatchObject({
+      threadId: "thread-2",
+      source: "ai",
+      comparisonMode: "responses",
+    });
+    expect(useExperimentStore.getState().experiments["experiment-1"]?.crown).not.toHaveProperty(
+      "snapshotHash",
+    );
+  });
+
   it("does not invoke an unavailable one-shot judge", async () => {
     useAppStore.setState((state) => ({
       ...state,

src/renderer/actions/experimentDecisionActions.ts

@@ -4,6 +4,7 @@ import {
   type CaptureExperimentSnapshotResult,
   type Experiment,
   type ExperimentCandidate,
+  type ExperimentJudgeMode,
   type Project,
   type ProjectLocation,
 } from "@/shared/contracts";
@@ -37,12 +38,14 @@ import {
 } from "./experimentWorktreeActions";
 import { runGitMergeToSource, showGitOperationFailure } from "./gitCommandRunner";
 import { applyDefaultPrAutomation } from "./prAutomationActions";
+import { buildExperimentResponseTranscript } from "./experimentResponseTranscript";
 
 export interface ExperimentJudgeSelection {
   agentKind: string;
   model: string;
   effort?: string;
   fast: boolean;
+  mode?: ExperimentJudgeMode;
 }
 
 function experimentSnapshotCandidates(experiment: Experiment) {
@@ -105,9 +108,17 @@ async function commitCandidateChanges(
 }
 
 export type ExperimentJudgeProgressEvent =
-  | { kind: "capturing" }
-  | { kind: "captured"; threadId: string; files: number; insertions: number; deletions: number }
-  | { kind: "judging" }
+  | { kind: "capturing"; mode: ExperimentJudgeMode }
+  | {
+      kind: "captured";
+      threadId: string;
+      files: number;
+      insertions: number;
+      deletions: number;
+      omittedFiles?: number;
+    }
+  | { kind: "captured-response"; threadId: string; characters: number }
+  | { kind: "judging"; mode: ExperimentJudgeMode }
   | {
       kind: "winner";
       threadId: string;
@@ -157,14 +168,28 @@ export async function crownExperiment(
     const judgeModel = selection?.model ?? judgeThread?.config.model;
     const judgeEffort = selection?.effort ?? judgeThread?.config.effort;
     const judgeFast = selection?.fast ?? judgeThread?.config.fast;
-    onProgress?.({ kind: "capturing" });
+    const judgeMode = selection?.mode ?? "changes";
+    onProgress?.({ kind: "capturing", mode: judgeMode });
     const unsubscribeProgress = readBridge().onSupervisorEvent((event) => {
       if (event.type !== "experiment-judge-progress" || event.experimentId !== experimentId) {
         return;
       }
-      onProgress?.(event.progress);
+      onProgress?.(
+        event.progress.kind === "judging" ? { kind: "judging", mode: judgeMode } : event.progress,
+      );
     });
     try {
+      const responses =
+        judgeMode === "responses"
+          ? await Promise.all(
+              experiment.candidates.map(async (candidate) => ({
+                threadId: candidate.threadId,
+                response: buildExperimentResponseTranscript(
+                  await readBridge().dbGetThreadRuntimeItems(candidate.threadId),
+                ),
+              })),
+            )
+          : undefined;
       const result = await readBridge().judgeExperimentSnapshot({
         experimentId,
         projectLocation: project.location,
@@ -173,6 +198,8 @@ export async function crownExperiment(
         ...(judgeModel ? { model: judgeModel } : {}),
         ...(judgeEffort ? { effort: judgeEffort } : {}),
         ...(judgeFast !== undefined ? { fast: judgeFast } : {}),
+        mode: judgeMode,
+        ...(responses ? { responses } : {}),
         prompt: experiment.prompt,
         candidates: experimentSnapshotCandidates(experiment),
       });
@@ -182,8 +209,9 @@ export async function crownExperiment(
         rationale: result.rationale,
         assessments: result.assessments,
         source: "ai",
+        comparisonMode: judgeMode,
         modelLabel: judgeLabel,
-        snapshotHash: result.hash,
+        ...(judgeMode === "changes" ? { snapshotHash: result.hash } : {}),
         createdAt: new Date().toISOString(),
       });
       captureProductEvent("experiment.winner_selected", {
@@ -199,6 +227,7 @@ export async function crownExperiment(
         candidate_count: experiment.candidates.length,
         provider: normalizeAnalyticsProvider(judgeAgent.kind),
         source: "ai",
+        comparison_mode: judgeMode,
       });
       onProgress?.({
         kind: "winner",

src/renderer/actions/experimentResponseTranscript.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import type { PersistedRuntimeItem } from "@/shared/ipc";
+import { buildExperimentResponseTranscript } from "./experimentResponseTranscript";
+
+describe("buildExperimentResponseTranscript", () => {
+  it("keeps top-level user and assistant messages while excluding tool and sub-agent rows", () => {
+    const items: PersistedRuntimeItem[] = [
+      {
+        id: "user",
+        type: "user_message",
+        state: "completed",
+        payload: { content: [{ kind: "text", text: "Research this" }] },
+        streams: {},
+      },
+      {
+        id: "tool",
+        type: "web_search",
+        state: "completed",
+        payload: { query: "example" },
+        streams: {},
+      },
+      {
+        id: "child",
+        type: "assistant_message",
+        state: "completed",
+        payload: { content: [{ kind: "text", text: "Hidden sub-agent output" }] },
+        streams: {},
+        parentItemId: "tool",
+      },
+      {
+        id: "assistant",
+        type: "assistant_message",
+        state: "completed",
+        streams: { assistant_text: "Final answer" },
+      },
+    ];
+
+    expect(buildExperimentResponseTranscript(items)).toBe(
+      "User:\nResearch this\n\nAssistant:\nFinal answer",
+    );
+  });
+});

src/renderer/actions/experimentResponseTranscript.ts

@@ -0,0 +1,49 @@
+import type { PersistedRuntimeItem } from "@/shared/ipc";
+import { MAX_EXPERIMENT_RESPONSE_LENGTH } from "@/shared/contracts";
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === "object" ? (value as Record<string, unknown>) : null;
+}
+
+function textFromContentBlocks(payload: unknown): string {
+  const content = asRecord(payload)?.content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .map((block) => {
+      const record = asRecord(block);
+      if (!record) return "";
+      if (record.kind === "text" && typeof record.text === "string") return record.text;
+      if (record.kind === "file" && typeof record.path === "string") return `@${record.path}`;
+      if (record.kind === "image") {
+        if (typeof record.path === "string") return `@${record.path}`;
+        if (typeof record.name === "string") return `[image: ${record.name}]`;
+        return "[image]";
+      }
+      return "";
+    })
+    .filter(Boolean)
+    .join("\n");
+}
+
+function formatChatMessage(item: PersistedRuntimeItem): string | null {
+  if (item.parentItemId) return null;
+  if (item.type === "user_message") {
+    const text = textFromContentBlocks(item.payload);
+    return text ? `User:\n${text}` : null;
+  }
+  if (item.type === "assistant_message") {
+    const text = textFromContentBlocks(item.payload) || item.streams.assistant_text;
+    return text ? `Assistant:\n${text}` : null;
+  }
+  return null;
+}
+
+export function buildExperimentResponseTranscript(items: readonly PersistedRuntimeItem[]): string {
+  const transcript = items
+    .map(formatChatMessage)
+    .filter((message): message is string => Boolean(message?.trim()))
+    .join("\n\n");
+  if (transcript.length <= MAX_EXPERIMENT_RESPONSE_LENGTH) return transcript;
+  const prefix = "[earlier chat truncated]\n\n";
+  return `${prefix}${transcript.slice(-(MAX_EXPERIMENT_RESPONSE_LENGTH - prefix.length))}`;
+}

src/renderer/i18n/sharedMessages.ts

@@ -150,6 +150,9 @@ const SHARED_MESSAGE_DESCRIPTORS: Record<MessageKey, MessageDescriptor> = {
   "experiment.judge.noChanges": msg({
     message: "The candidates have not made any changes yet.",
   }),
+  "experiment.judge.noResponse": msg({
+    message: "No chat response is available for experiment candidate {threadId}.",
+  }),
   "experiment.judge.promptBlank": msg({ message: "Experiment prompt must not be blank" }),
   "experiment.judge.uniqueThreadIds": msg({
     message: "Experiment candidate thread ids must be unique",