Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions evalboard/app/_components/harness-badge.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import Image from "next/image";

// Vendor logo for a run's harness (RunConfig). Renders the recognizable
// vendor mark instead of raw "claude-code"/"codex"/"antigravity" text,
// mirroring the Slack rollup's vendor emoji. A missing harness defaults to
// claude-code (the nightly). This is an internal-only column — the caller
// gates it behind isInternal (see lib/edition.ts).
const HARNESS_LOGO: Record<string, { src: string; label: string }> = {
"claude-code": {
src: "/harness/claude-code.png",
label: "Claude Code · Anthropic",
},
codex: { src: "/harness/codex.png", label: "Codex · OpenAI" },
antigravity: {
src: "/harness/antigravity.png",
label: "Antigravity · Google Gemini",
},
};

export function HarnessBadge({ harness }: { harness?: string | null }) {
const key = harness ?? "claude-code";
const logo = HARNESS_LOGO[key];
// Unknown harness: show the raw id rather than a misleading logo.
if (!logo) {
return <span className="text-xs text-gray-700">{key}</span>;
}
return (
<Image
src={logo.src}
alt={logo.label}
title={logo.label}
width={20}
height={20}
className="rounded-sm"
/>
);
}
15 changes: 14 additions & 1 deletion evalboard/app/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { ChipLegend, MergedTagRail } from "./_overview/tag-rail";
import { TableScroll } from "./_components/scroll-table";
import { CollapsibleRail } from "./_components/collapsible-rail";
import { isInternal } from "@/lib/edition";
import { HarnessBadge } from "@/app/_components/harness-badge";

export const dynamic = "force-dynamic";

Expand Down Expand Up @@ -330,6 +331,11 @@ export default async function Page({
<thead>
<tr className="bg-gray-50 border-b border-gray-200 text-left text-gray-600">
<th className="py-3 px-4 font-medium">Run</th>
{isInternal && (
<th className="py-3 px-4 font-medium">
Harness
</th>
)}
<th className="py-3 px-4 font-medium">
Pass rate
</th>
Expand Down Expand Up @@ -363,6 +369,13 @@ export default async function Page({
{fmtRunTime(r.id)}
</Link>
</td>
{isInternal && (
<td className="py-3 px-4">
<HarnessBadge
harness={r.harness}
/>
</td>
)}
<td className="py-3 px-4 tabular-nums">
<span
className={`font-medium ${passClass(
Expand Down Expand Up @@ -393,7 +406,7 @@ export default async function Page({
{listing.rows.length === 0 && (
<tr>
<td
colSpan={5}
colSpan={isInternal ? 6 : 5}
className="py-6 px-4 text-center text-sm text-gray-500"
>
{isFiltered
Expand Down
6 changes: 3 additions & 3 deletions evalboard/lib/__tests__/pricing-parity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ import { describe, expect, test } from "vitest";
import { PRICING } from "../pricing";

// Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative
// Python table in src/coder_eval/proxy/pricing.py. If the backend reprices a
// Python table in src/coder_eval/pricing.py. If the backend reprices a
// model (or adds one) and this mirror isn't updated, the frontend's "estimated"
// USD figures silently disagree with the backend's authoritative Cost on the
// same tokens. This test parses the Python table and asserts both tables have
// the same model ids and the same four per-MTok rates — turning silent drift
// into a failing build.

const here = dirname(fileURLToPath(import.meta.url));
const PY_PATH = resolve(here, "../../../src/coder_eval/proxy/pricing.py");
const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py");

// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125),
const ROW_RE =
Expand All @@ -36,7 +36,7 @@ function parsePythonTable(): Record<
return out;
}

describe("pricing.ts ↔ proxy/pricing.py parity", () => {
describe("pricing.ts ↔ pricing.py parity", () => {
const py = parsePythonTable();

test("parses a non-trivial Python table", () => {
Expand Down
36 changes: 36 additions & 0 deletions evalboard/lib/__tests__/runs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
type ArtifactRef,
clearRunCacheDir,
extractComponentShas,
extractRunConfig,
findMatureSourceRuns,
isExcludedArtifact,
type MessageEvent,
Expand Down Expand Up @@ -346,6 +347,41 @@ describe("extractComponentShas", () => {
});
});

describe("extractRunConfig", () => {
test("prefers the run-level RunConfig stamp (harness + environment)", () => {
const out = extractRunConfig({
environment_info: {
run_config: {
harness: "codex",
model: "gpt-5.4",
environment: "prod",
},
},
});
expect(out.harness).toBe("codex");
expect(out.environment).toBe("prod");
});

test("falls back to the most common per-task agent_config.type", () => {
const out = extractRunConfig({
task_results: [
{ task_id: "a", agent_config: { type: "antigravity" } },
{ task_id: "b", agent_config: { type: "antigravity" } },
{ task_id: "c", agent_config: { type: "claude-code" } },
],
});
expect(out.harness).toBe("antigravity");
// environment is only known from the run-level stamp, never derived.
expect(out.environment).toBeNull();
});

test("null harness when nothing identifies it (legacy runs)", () => {
const out = extractRunConfig({ task_results: [{ task_id: "a" }] });
expect(out.harness).toBeNull();
expect(out.environment).toBeNull();
});
});

describe("findMatureSourceRuns", () => {
// Newest-first run list with injected run.json readers (the deps test seam).
const ids = ["r5", "r4", "r3", "r2", "r1"];
Expand Down
6 changes: 6 additions & 0 deletions evalboard/lib/overview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ export interface RunListingRow {
tasksRun: number;
totalCostUsd: number | null;
taskDurationSeconds: number | null;
// Run-level harness (coder-eval AgentKind) for the Harness column; null on
// legacy runs that predate the RunConfig stamp / carry no agent_config.type.
// Optional so existing test factories stay valid.
harness?: string | null;
}

// Window-level rollup across every matched run (pre-limit), so the front-page
Expand Down Expand Up @@ -680,6 +684,7 @@ export async function getRunListing(
tasksRun: scopedTasks.length,
totalCostUsd: scopedCost,
taskDurationSeconds: scopedDur,
harness: overview.harness ?? null,
});
}

Expand Down Expand Up @@ -735,6 +740,7 @@ export function buildAdhocRows(
tasksRun: overview.tasks.length,
totalCostUsd: overview.totalCostUsd,
taskDurationSeconds: overview.taskDurationSeconds,
// Harness is a main-table-only (internal) column; ad-hoc rows omit it.
});
}
rows.sort(
Expand Down
15 changes: 12 additions & 3 deletions evalboard/lib/pricing.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Per-million-token prices and cost math. Ported from
// src/coder_eval/proxy/pricing.py — keep in sync when that table changes.
// Source: Anthropic / OpenAI public pricing.
// src/coder_eval/pricing.py — keep in sync when that table changes.
// Source: Anthropic / OpenAI / Google public pricing.
//
// This is the single source of truth for rates on the frontend: the
// cascade-aware thinking-cost simulator (lib/thinkingSim.ts) and the
Expand All @@ -15,7 +15,7 @@ export interface Pricing {
}

// Exported so a unit test can assert key-and-rate parity against the
// authoritative Python table (src/coder_eval/proxy/pricing.py) and fail the
// authoritative Python table (src/coder_eval/pricing.py) and fail the
// build on drift — this hand-copied mirror is otherwise guarded only by a
// comment. Not part of the consumer API; use resolvePricing() instead.
export const PRICING: Record<string, Pricing> = {
Expand All @@ -41,6 +41,15 @@ export const PRICING: Record<string, Pricing> = {
"gpt-5.3-codex": p(1.75, 14, 1.75, 0.175),
"gpt-5.4": p(2.5, 15, 2.5, 0.25),
"gpt-5.5": p(5, 30, 5, 0.5),
// Google Gemini (AntigravityAgent). Gemini bills no separate cache-write
// fee (cache_write == input, effectively unused); cache_read is the cached-
// input rate. Pro's >200K-token tier is higher — this flat rate reads low
// for very-large-context runs, fine for typical eval tasks.
"gemini-3-pro-preview": p(2, 12, 2, 0.2),
"gemini-3.1-pro-preview": p(2, 12, 2, 0.2),
"gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2),
"gemini-3.5-flash": p(1.5, 9, 1.5, 0.15),
"gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15),
};

function p(
Expand Down
62 changes: 62 additions & 0 deletions evalboard/lib/runs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,10 @@ interface RawTaskResult {
// Model the task ran on (e.g. "claude-sonnet-4-6"). Used to price token
// buckets as USD. Absent on legacy runs.
model_used?: string | null;
// Per-task agent config; `type` is the harness (coder-eval AgentKind, e.g.
// "claude-code" | "codex" | "antigravity"). Used to derive the run's harness
// when the run-level RunConfig stamp is absent (direct coder-eval / legacy runs).
agent_config?: { type?: string | null } | null;
// Activation enrichment the dashboard folds onto each case row in the nested
// activation run.json (see cli.py _finalize_activation_run). Absent on skills
// rows. `prompt` = the case's prompt text; `expected_skill` = the skill the
Expand Down Expand Up @@ -790,6 +794,12 @@ export interface RunOverview {
totalCostUsd: number | null;
taskDurationSeconds: number | null;
componentShas: ComponentSha[];
// Run-level harness (coder-eval AgentKind) from the RunConfig stamp
// (environment_info.run_config), falling back to the most common per-task
// agent_config.type. `environment` (alpha/prod) is captured but intentionally
// NOT surfaced in the UI yet. Both optional so test factories predating them stay valid.
harness?: string | null;
environment?: string | null;
// run.json `start_time` (ISO wall-clock). Pipeline runs derive their date
// from the date-shaped id; ad-hoc ids carry no date, so the ad-hoc listing
// orders by this instead. Optional so test factories predating it stay valid.
Expand All @@ -814,6 +824,57 @@ export function visibleTurnsFromRaw(t: {
return null;
}

// The run's harness + environment. Prefers the run-level RunConfig stamped by
// eval_runner into environment_info.run_config; falls back to the most common
// per-task agent_config.type (direct coder-eval / legacy runs carry no stamp).
// Returns null harness when nothing identifies it (legacy pre-harness runs).
export function extractRunConfig(data: RawRunJson): {
harness: string | null;
environment: string | null;
} {
const rc = data.environment_info?.run_config;
if (rc && typeof rc === "object" && !Array.isArray(rc)) {
const r = rc as Record<string, unknown>;
return {
// A present-but-partial stamp (run_config object with no/empty
// harness) falls back to the per-task agent_config vote rather
// than nulling out — otherwise a real codex/antigravity run is
// mislabeled as the claude-code default.
harness:
typeof r.harness === "string" && r.harness
? r.harness
: mostCommonAgentType(data.task_results ?? []),
environment:
typeof r.environment === "string" ? r.environment : null,
};
}
return {
harness: mostCommonAgentType(data.task_results ?? []),
environment: null,
};
}

// Most frequent per-task agent_config.type across a run's rows, or null if none
// declare one. A run is single-harness in practice, so this is just a robust
// "the harness these tasks ran on" for runs without the run-level stamp.
function mostCommonAgentType(rows: RawTaskResult[]): string | null {
const counts = new Map<string, number>();
for (const r of rows) {
const t = r.agent_config?.type;
if (typeof t === "string" && t)
counts.set(t, (counts.get(t) ?? 0) + 1);
}
let best: string | null = null;
let bestN = 0;
for (const [t, n] of counts) {
if (n > bestN) {
best = t;
bestN = n;
}
}
return best;
}

export async function readRunOverview(
id: string,
): Promise<RunOverview | null> {
Expand Down Expand Up @@ -859,6 +920,7 @@ export async function readRunOverview(
? taskDurationSum
: (data.total_duration_seconds ?? null),
componentShas: extractComponentShas(data.environment_info),
...extractRunConfig(data),
startedAt: data.start_time ?? null,
};
}
Expand Down
Binary file added evalboard/public/harness/antigravity.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added evalboard/public/harness/claude-code.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added evalboard/public/harness/codex.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading