diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index a16655015..7fb3bbc0d 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1344,7 +1344,8 @@ async function prepareFileMetadata(params: { selections = multiSelections.map((sel) => ({ selection: sel, - inlineTargetLabel: resolveTargetLabel(sel.targetName, sel.resolvedTarget.name), + inlineTargetLabel: + sel.targetLabel ?? resolveTargetLabel(sel.targetName, sel.resolvedTarget.name), })); } else { // Single target mode (legacy path) @@ -1368,18 +1369,22 @@ async function prepareFileMetadata(params: { }); // Attach target hooks from eval file if available - const singleTargetHooks = targetRefs?.find((ref) => ref.name === selection.targetName)?.hooks; - const augmentedSelection: TargetSelection = singleTargetHooks - ? { ...selection, targetHooks: singleTargetHooks } - : selection; + const singleTargetRef = targetRefs?.find((ref) => ref.name === selection.targetName); + const augmentedSelection: TargetSelection = { + ...selection, + ...(singleTargetRef?.label ? { targetLabel: singleTargetRef.label } : {}), + ...(singleTargetRef?.hooks ? { targetHooks: singleTargetRef.hooks } : {}), + }; selections = [ { selection: augmentedSelection, - inlineTargetLabel: resolveTargetLabel( - augmentedSelection.targetName, - augmentedSelection.resolvedTarget.name, - ), + inlineTargetLabel: + augmentedSelection.targetLabel ?? + resolveTargetLabel( + augmentedSelection.targetName, + augmentedSelection.resolvedTarget.name, + ), }, ]; } @@ -2307,7 +2312,8 @@ export async function runEvalCommand( const explicitVariant = targetVariantForSelection(selection); const skippedResults: EvaluationResult[] = targetPrep.testCases.map((testCase) => ({ timestamp: new Date().toISOString(), - testId: testCase.id, + testId: testCase.testId ?? testCase.id, + prompt: testCase.prompt, score: 0, assertions: [], output: budgetMsg, @@ -2316,7 +2322,7 @@ export async function runEvalCommand( output: [{ role: 'assistant' as const, content: budgetMsg }], finalOutput: budgetMsg, target: selection.targetName, - testId: testCase.id, + testId: testCase.testId ?? testCase.id, conversationId: testCase.conversation_id, error: budgetMsg, }), @@ -2426,7 +2432,8 @@ export async function runEvalCommand( withSourceMetadata( { timestamp: new Date().toISOString(), - testId: testCase.id, + testId: testCase.testId ?? testCase.id, + prompt: testCase.prompt, score: 0, assertions: [], output: message, @@ -2435,7 +2442,7 @@ export async function runEvalCommand( output: [{ role: 'assistant' as const, content: message }], finalOutput: message, target: selection.targetName, - testId: testCase.id, + testId: testCase.testId ?? testCase.id, conversationId: testCase.conversation_id, error: message, }), diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 7621e6ebe..8ad08b952 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -85,6 +85,7 @@ export interface TargetSelection { readonly definitions: readonly TargetDefinition[]; readonly resolvedTarget: ResolvedTarget; readonly targetName: string; + readonly targetLabel?: string; readonly targetSource: 'cli' | 'test-file' | 'default'; readonly targetsFilePath: string; /** Per-target hooks from eval file (eval-level customization) */ @@ -260,11 +261,15 @@ export async function selectMultipleTargets( // Build a lookup for target hooks from eval target refs const hooksMap = new Map(); + const labelsMap = new Map(); if (targetRefs) { for (const ref of targetRefs) { if (ref.hooks) { hooksMap.set(ref.name, ref.hooks); } + if (ref.label) { + labelsMap.set(ref.name, ref.label); + } } } @@ -323,6 +328,7 @@ export async function selectMultipleTargets( modelOverride, ); const hooks = hooksMap.get(name); + const targetLabel = labelsMap.get(name); try { const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath, { @@ -332,6 +338,7 @@ export async function selectMultipleTargets( definitions, resolvedTarget, targetName: name, + ...(targetLabel ? { targetLabel } : {}), targetSource: options.targetSource ?? 'cli', targetsFilePath, ...(hooks && { targetHooks: hooks }), diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 790c810a3..42ec16517 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -12,6 +12,8 @@ import { type GraderResult, METRICS_SCHEMA_VERSION, MetricsArtifactWireSchema, + buildEvalTestTargetKey, + buildEvaluationResultTargetKey, buildResultIndexArtifact, buildTraceFromMessages, parseYamlValue, @@ -1304,6 +1306,31 @@ describe('writeArtifactsFromResults', () => { expect(indexEntry?.trials?.[1]?.transcript_summary).toEqual(runTwoResult.transcript_summary); }); + it('keys prompt-expanded resume checks by authored test id plus prompt id', () => { + const prompt = { id: 'direct', label: 'Direct prompt', kind: 'string' as const }; + const completed = makeResult({ + testId: 'docs', + prompt, + target: 'mock-target', + }); + const expandedTest = { + id: 'docs__prompt_direct', + testId: 'docs', + prompt, + input: [{ role: 'user', content: 'Prompt text' }], + expected_output: [], + reference_answer: '', + file_paths: [], + criteria: 'ok', + evaluator: 'llm-grader', + assertions: [], + } as unknown as EvalTest; + + expect(buildEvalTestTargetKey(expandedTest, 'mock-target')).toBe( + buildEvaluationResultTargetKey(completed), + ); + }); + it('handles empty results array', async () => { const paths = await writeArtifactsFromResults([], testDir); diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts new file mode 100644 index 000000000..f8b1aaa09 --- /dev/null +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -0,0 +1,66 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { loadTestSuite } from '@agentv/core'; + +import { selectMultipleTargets } from '../../../src/commands/eval/targets.js'; + +describe('eval target selection', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), 'agentv-target-selection-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('resolves authored target ids through targets.yaml while keeping labels for display', async () => { + const agentvDir = path.join(tempDir, '.agentv'); + await mkdir(agentvDir, { recursive: true }); + await writeFile( + path.join(agentvDir, 'targets.yaml'), + [ + '$schema: agentv-targets-v2.2', + 'targets:', + ' - label: openai:gpt-5.4-mini', + ' provider: mock', + '', + ].join('\n'), + ); + const evalPath = path.join(tempDir, 'target-label.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: target-label-suite', + 'targets:', + ' - id: openai:gpt-5.4-mini', + ' label: mini', + 'tests:', + ' - id: target-case', + ' input: hello', + ' criteria: ok', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + const selections = await selectMultipleTargets({ + testFilePath: evalPath, + repoRoot: tempDir, + cwd: tempDir, + env: {}, + targetNames: suite.targets ?? [], + targetRefs: suite.targetRefs, + targetSource: 'test-file', + }); + + expect(selections).toHaveLength(1); + expect(selections[0]?.targetName).toBe('openai:gpt-5.4-mini'); + expect(selections[0]?.targetLabel).toBe('mini'); + expect(selections[0]?.resolvedTarget.kind).toBe('mock'); + }); +}); diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index 1c7e5e6a9..ed850db91 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -255,7 +255,7 @@ describe('agentv prepare', () => { path.join(tempDir, '.agentv', 'targets.yaml'), ` targets: - - name: codex + - label: codex provider: cli command: bun ./scripts/target.ts `, diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index ab45dbe00..d18add351 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -398,25 +398,43 @@ function parseEvalTargetRef(raw: unknown, location: string): EvalTargetRef { } const rawLabel = raw.label; + const rawId = raw.id; + const useTarget = raw.use_target; + const legacyName = raw.name; + const id = typeof rawId === 'string' && rawId.trim().length > 0 ? rawId.trim() : undefined; const label = typeof rawLabel === 'string' && rawLabel.trim().length > 0 ? rawLabel.trim() : undefined; - if (!label) { - throw new Error(`Invalid ${location}: target object requires a 'label' field.`); + const useTargetName = + typeof useTarget === 'string' && useTarget.trim().length > 0 ? useTarget.trim() : undefined; + const legacyTargetName = + typeof legacyName === 'string' && legacyName.trim().length > 0 ? legacyName.trim() : undefined; + if (legacyName !== undefined) { + throw new Error( + `Invalid ${location}: target field 'name' has been removed. Use 'id' and 'label' instead.`, + ); } const hooks = parseTargetHooks(raw.hooks); - const definition = normalizeTargetDefinition( - Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')), - ) as TargetDefinition; - const useTarget = - typeof raw.use_target === 'string' && raw.use_target.trim().length > 0 - ? raw.use_target.trim() - : undefined; + const hasInlineDefinition = typeof raw.provider === 'string' || useTargetName !== undefined; + if (hasInlineDefinition && !label) { + throw new Error(`Invalid ${location}: target object requires a 'label' field.`); + } + const name = hasInlineDefinition ? label : (id ?? legacyTargetName ?? label); + if (!name) { + throw new Error(`Invalid ${location}: target object requires an 'id' or 'label' field.`); + } + const definition = hasInlineDefinition + ? (normalizeTargetDefinition( + Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')), + ) as TargetDefinition) + : undefined; return { - name: label, - ...(useTarget !== undefined ? { use_target: useTarget } : {}), - definition, + name, + ...(id !== undefined ? { id } : {}), + ...(label !== undefined ? { label } : {}), + ...(useTargetName !== undefined ? { use_target: useTargetName } : {}), + ...(definition ? { definition } : {}), ...(hooks !== undefined ? { hooks } : {}), }; } diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 29518ee3a..c72e3ef22 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -665,7 +665,7 @@ export async function gradePreparedEvalCase( finalOutput: candidate, provider: provider.kind, target: target.name, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, }); @@ -710,7 +710,8 @@ export async function gradePreparedEvalCase( : classifyQualityStatus(score.score, effectiveThreshold); const baseResult = { timestamp: timestamp.toISOString(), - testId: evalCase.id, + testId: authoredResultTestId(evalCase), + prompt: evalCase.prompt, source: evalCase.source, suite: evalCase.suite, category: evalCase.category, @@ -1077,9 +1078,10 @@ export async function runEvaluation( const errorMessage = `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`; const budgetResult: EvaluationResult = { timestamp: (now ?? (() => new Date()))().toISOString(), - testId: evalCase.id, + testId: authoredResultTestId(evalCase), suite: evalCase.suite, category: evalCase.category, + prompt: evalCase.prompt, score: 0, assertions: [], output: errorMessage, @@ -1088,7 +1090,7 @@ export async function runEvaluation( output: [{ role: 'assistant' as const, content: errorMessage }], finalOutput: errorMessage, target: target.name, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, error: errorMessage, }), @@ -1126,9 +1128,10 @@ export async function runEvaluation( const errorMessage = `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${budgetUsd.toFixed(4)})`; const budgetResult: EvaluationResult = { timestamp: (now ?? (() => new Date()))().toISOString(), - testId: evalCase.id, + testId: authoredResultTestId(evalCase), suite: evalCase.suite, category: evalCase.category, + prompt: evalCase.prompt, score: 0, assertions: [], output: errorMessage, @@ -1137,7 +1140,7 @@ export async function runEvaluation( output: [{ role: 'assistant' as const, content: errorMessage }], finalOutput: errorMessage, target: target.name, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, error: errorMessage, }), @@ -1175,9 +1178,10 @@ export async function runEvaluation( const errorMsg = 'Halted: execution error encountered with fail_on_error enabled'; const haltResult: EvaluationResult = { timestamp: (now ?? (() => new Date()))().toISOString(), - testId: evalCase.id, + testId: authoredResultTestId(evalCase), suite: evalCase.suite, category: evalCase.category, + prompt: evalCase.prompt, score: 0, assertions: [], output: errorMsg, @@ -1186,7 +1190,7 @@ export async function runEvaluation( output: [{ role: 'assistant' as const, content: errorMsg }], finalOutput: errorMsg, target: target.name, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, error: errorMsg, }), @@ -1374,9 +1378,10 @@ export async function runEvaluation( const errorMsg = `${prefix}: dependency failed (${failedDeps.join(', ')})`; const depFailResult: EvaluationResult = { timestamp: (now ?? (() => new Date()))().toISOString(), - testId: evalCase.id, + testId: authoredResultTestId(evalCase), suite: evalCase.suite, category: evalCase.category, + prompt: evalCase.prompt, score: 0, assertions: [], output: errorMsg, @@ -1385,7 +1390,7 @@ export async function runEvaluation( output: [{ role: 'assistant' as const, content: errorMsg }], finalOutput: errorMsg, target: target.name, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, error: errorMsg, }), @@ -2598,7 +2603,7 @@ async function evaluateCandidate(options: { endTime, provider: provider.kind, target: target.name, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, }); @@ -3571,6 +3576,10 @@ async function invokeProvider( } } +function authoredResultTestId(evalCase: Pick): string { + return evalCase.testId ?? evalCase.id; +} + function buildErrorResult( evalCase: EvalTest, targetName: string, @@ -3620,14 +3629,15 @@ function buildErrorResult( output: [{ role: 'assistant' as const, content: output }], finalOutput: output, target: targetName, - testId: evalCase.id, + testId: authoredResultTestId(evalCase), conversationId: evalCase.conversation_id, error: message, }); return { timestamp: timestamp.toISOString(), - testId: evalCase.id, + testId: authoredResultTestId(evalCase), + prompt: evalCase.prompt, suite: evalCase.suite, category: evalCase.category, conversationId: evalCase.conversation_id, diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index f8a6fc941..470c82fcc 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -121,20 +121,22 @@ export function buildEvaluationResultTargetKey(result: EvaluationResult): string null, suite: stringField(dimensions, 'suite') ?? getSuite(result) ?? null, test_id: stringField(dimensions, 'testId') ?? result.testId ?? 'unknown', + prompt_id: stringField(dimensions, 'promptId') ?? result.prompt?.id ?? null, target: stringField(dimensions, 'target') ?? result.target ?? 'unknown', variant: stringField(dimensions, 'variant') ?? result.variant ?? null, }); } export function buildEvalTestTargetKey( - test: Pick, + test: Pick, target?: string, variant?: string, ): string { return JSON.stringify({ eval_path: evalSourcePath(test.source) ?? null, suite: test.suite ?? null, - test_id: test.id ?? 'unknown', + test_id: test.testId ?? test.id ?? 'unknown', + prompt_id: test.prompt?.id ?? null, target: target ?? 'unknown', variant: variant ?? null, }); diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 806d86146..3f307f3dc 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -324,6 +324,10 @@ export type AgentVExtensionConfig = AgentRulesExtensionConfig | FileExtensionCon export type EvalTargetRef = { /** Internal target identity (authored as `label` in object form). */ readonly name: string; + /** Provider/backend locator identity from authored eval YAML. */ + readonly id?: string; + /** Display/comparison label from authored eval YAML. */ + readonly label?: string; /** Delegate to another named target (same as use_target in targets.yaml) */ readonly use_target?: string; /** Inline target definition normalized from a promptfoo-shaped target object. */ @@ -1001,14 +1005,30 @@ export type ConversationAggregation = 'mean' | 'min' | 'max'; */ export type TurnFailurePolicy = 'continue' | 'stop'; +export type EvalPromptKind = 'string' | 'chat' | 'file' | 'function'; + +/** + * Stable identity for an authored top-level prompt. The prompt content itself + * is rendered into EvalTest.input; this metadata keeps the matrix dimension + * visible to reports, artifacts, and future flat-instance workers. + */ +export interface EvalPromptIdentity { + readonly id: string; + readonly label?: string; + readonly kind: EvalPromptKind; +} + /** * Eval test definition sourced from AgentV specs. */ export interface EvalTest { readonly id: string; + /** Original authored test id before prompt expansion rewrites duplicate internal ids. */ + readonly testId?: string; readonly suite?: string; readonly category?: string; readonly conversation_id?: string; + readonly prompt?: EvalPromptIdentity; readonly question: string; readonly input: readonly TestMessage[]; readonly expected_output: readonly JsonObject[]; @@ -1195,6 +1215,11 @@ export type FailOnError = boolean; export interface EvaluationResult { readonly timestamp: string; readonly testId: string; + readonly prompt?: EvalPromptIdentity; + /** Zero-based sample index produced from repeat.count. */ + readonly sampleIndex?: number; + /** Provider retry index for the attempt that produced this result. */ + readonly retryIndex?: number; readonly source?: EvalTestSource; readonly suite?: string; readonly category?: string; diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 021cece7d..583d5a73d 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -68,6 +68,8 @@ const PromptSchema = z.union([ id: z.string().optional(), label: z.string().optional(), raw: z.string().optional(), + function: z.string().optional(), + function_file: z.string().optional(), path: z.string().optional(), prefix: z.string().optional(), suffix: z.string().optional(), diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 2914ce8df..7d0f2e2ad 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFile, realpath, stat } from 'node:fs/promises'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -11,6 +12,7 @@ import { normalizeExperimentConfig, normalizeExperimentRunOverride, } from './experiment.js'; +import { executeScript } from './graders/code-grader.js'; import { collectResolvedInputFilePaths } from './input-message-utils.js'; import { type NunjucksFilterMap, @@ -36,7 +38,11 @@ import { loadConfig, parseTargetHooks, } from './loaders/config-loader.js'; -import { buildSearchRoots, resolveToAbsolutePath } from './loaders/file-resolver.js'; +import { + buildSearchRoots, + resolveFileReference, + resolveToAbsolutePath, +} from './loaders/file-resolver.js'; import { coerceEvaluator, collectAssertionTemplateSourceReferences, @@ -65,6 +71,7 @@ import type { ConversationTurn, DockerWorkspaceConfig, EvalGraderSource, + EvalPromptIdentity, EvalRunOverride, EvalSourceReference, EvalTest, @@ -252,6 +259,17 @@ type RawEvalCase = JsonObject & { readonly window_size?: JsonValue; }; +type PromptDefinition = { + readonly identity: EvalPromptIdentity; + readonly input: JsonValue; +}; + +type PromptExpansionResult = { + readonly rawCases: readonly JsonValue[]; + readonly promptById: ReadonlyMap; + readonly sourceTestIdById: ReadonlyMap; +}; + function resolveTests(suite: RawTestSuite): JsonValue | undefined { if (suite.tests !== undefined) return suite.tests; if (suite.eval_cases !== undefined) { @@ -369,6 +387,404 @@ function expandArrayVarCases(raw: RawEvalCase): readonly RawEvalCase[] { return combinations.map((vars) => ({ ...raw, vars })); } +function stablePromptId(value: unknown): string { + return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 12); +} + +function safePromptId(value: string): string { + const safe = value + .trim() + .replace(/[^A-Za-z0-9_.-]+/g, '-') + .replace(/^-+|-+$/g, ''); + return safe.length > 0 ? safe.slice(0, 48) : stablePromptId(value); +} + +function stripFileProtocol(value: string): string { + return value.startsWith('file://') ? value.slice('file://'.length) : value; +} + +function isChatPromptArray(value: readonly JsonValue[]): boolean { + return value.length > 0 && value.every((entry) => isJsonObject(entry) && isTestMessage(entry)); +} + +async function readPromptFile( + rawPath: string, + searchRoots: readonly string[], +): Promise<{ + readonly displayPath: string; + readonly text: string; +}> { + const filePath = stripFileProtocol(rawPath); + const { displayPath, resolvedPath, attempted } = await resolveFileReference( + filePath, + searchRoots, + ); + if (!resolvedPath) { + const attempts = attempted.length + ? [' Tried:', ...attempted.map((candidate) => ` ${candidate}`)] + : undefined; + logError(`Prompt file not found: ${displayPath}`, attempts); + throw new Error(`Prompt file not found: ${displayPath}`); + } + return { + displayPath, + text: (await readFile(resolvedPath, 'utf8')).replace(/\r\n/g, '\n'), + }; +} + +function promptSourceInputFromStdout(stdout: string, index: number): JsonValue { + const trimmed = stdout.trim(); + if (!trimmed) { + throw new Error(`Invalid prompts[${index}] function source: command produced empty output.`); + } + + try { + const parsed = JSON.parse(trimmed) as JsonValue; + if (typeof parsed === 'string' || (Array.isArray(parsed) && isChatPromptArray(parsed))) { + return parsed; + } + if (isJsonObject(parsed)) { + if (typeof parsed.prompt === 'string') { + return parsed.prompt; + } + if (typeof parsed.raw === 'string') { + return parsed.raw; + } + if (Array.isArray(parsed.messages) && isChatPromptArray(parsed.messages)) { + return parsed.messages; + } + } + } catch { + return trimmed; + } + + throw new Error( + `Invalid prompts[${index}] function source output: expected text or chat messages.`, + ); +} + +async function resolvePromptCommand( + command: readonly string[], + searchRoots: readonly string[], +): Promise { + const last = command.at(-1); + if (!last) { + return command; + } + + const resolved = await resolveFileReference(last, searchRoots); + return resolved.resolvedPath + ? [...command.slice(0, -1), path.resolve(resolved.resolvedPath)] + : command; +} + +async function executePromptSource( + command: readonly string[], + searchRoots: readonly string[], + index: number, +): Promise { + const resolvedCommand = await resolvePromptCommand(command, searchRoots); + const cwd = searchRoots[0] ? path.resolve(searchRoots[0]) : undefined; + try { + const stdout = await executeScript(resolvedCommand, '', undefined, cwd); + return promptSourceInputFromStdout(stdout, index); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Prompt function source failed for prompts[${index}]: ${message}`); + } +} + +async function executePromptFunctionFile( + rawPath: string, + searchRoots: readonly string[], + index: number, +): Promise<{ readonly displayPath: string; readonly input: JsonValue }> { + const filePath = stripFileProtocol(rawPath); + const { displayPath, resolvedPath, attempted } = await resolveFileReference( + filePath, + searchRoots, + ); + if (!resolvedPath) { + const attempts = attempted.length + ? [' Tried:', ...attempted.map((candidate) => ` ${candidate}`)] + : undefined; + logError(`Prompt function file not found: ${displayPath}`, attempts); + throw new Error(`Prompt function file not found: ${displayPath}`); + } + + return { + displayPath, + input: await executePromptSource( + [process.execPath, path.resolve(resolvedPath)], + searchRoots, + index, + ), + }; +} + +async function parsePromptDefinition( + rawPrompt: JsonValue, + searchRoots: readonly string[], + index: number, +): Promise { + if (typeof rawPrompt === 'string') { + if (rawPrompt.startsWith('file://')) { + const { displayPath, text } = await readPromptFile(rawPrompt, searchRoots); + return { + identity: { id: displayPath, label: displayPath, kind: 'file' }, + input: text, + }; + } + return { + identity: { id: `prompt-${stablePromptId(rawPrompt)}`, kind: 'string' }, + input: rawPrompt, + }; + } + + if (Array.isArray(rawPrompt)) { + if (!isChatPromptArray(rawPrompt)) { + throw new Error( + 'Invalid prompts entry: arrays must be chat messages or a top-level list of prompt entries.', + ); + } + return { + identity: { id: `chat-${stablePromptId(rawPrompt)}`, kind: 'chat' }, + input: rawPrompt, + }; + } + + if (!isJsonObject(rawPrompt)) { + throw new Error(`Invalid prompts[${index}]: expected string, chat array, or object.`); + } + + const label = asString(rawPrompt.label)?.trim(); + const explicitId = asString(rawPrompt.id)?.trim(); + + if (rawPrompt.function_file !== undefined) { + const functionFile = asString(rawPrompt.function_file); + if (!functionFile) { + throw new Error(`Invalid prompts[${index}].function_file: expected non-empty string.`); + } + const { displayPath, input } = await executePromptFunctionFile( + functionFile, + searchRoots, + index, + ); + return { + identity: { + id: explicitId ?? displayPath, + ...(label ? { label } : { label: displayPath }), + kind: 'function', + }, + input, + }; + } + + if (rawPrompt.function !== undefined) { + const functionSource = asString(rawPrompt.function); + if (!functionSource) { + throw new Error(`Invalid prompts[${index}].function: expected non-empty string path.`); + } + const { displayPath, input } = await executePromptFunctionFile( + functionSource, + searchRoots, + index, + ); + return { + identity: { + id: explicitId ?? displayPath, + ...(label ? { label } : { label: displayPath }), + kind: 'function', + }, + input, + }; + } + + if (rawPrompt.command !== undefined) { + const command = parseCommandArray(rawPrompt.command); + if (!command) { + throw new Error(`Invalid prompts[${index}].command: expected command string or array.`); + } + return { + identity: { + id: explicitId ?? `function-${stablePromptId(command)}`, + ...(label ? { label } : {}), + kind: 'function', + }, + input: await executePromptSource(command, searchRoots, index), + }; + } + + if (rawPrompt.file !== undefined) { + const fileRef = asString(rawPrompt.file); + if (!fileRef) { + throw new Error(`Invalid prompts[${index}].file: expected non-empty string.`); + } + const { displayPath, text } = await readPromptFile(fileRef, searchRoots); + return { + identity: { + id: explicitId ?? displayPath, + ...(label ? { label } : { label: displayPath }), + kind: 'file', + }, + input: text, + }; + } + + if (rawPrompt.messages !== undefined) { + if (!Array.isArray(rawPrompt.messages) || !isChatPromptArray(rawPrompt.messages)) { + throw new Error(`Invalid prompts[${index}].messages: expected chat message array.`); + } + return { + identity: { + id: explicitId ?? `chat-${stablePromptId(rawPrompt.messages)}`, + ...(label ? { label } : {}), + kind: 'chat', + }, + input: rawPrompt.messages, + }; + } + + if (rawPrompt.prompt !== undefined) { + const promptValue = rawPrompt.prompt; + if ( + typeof promptValue !== 'string' && + !(Array.isArray(promptValue) && isChatPromptArray(promptValue)) + ) { + throw new Error(`Invalid prompts[${index}].prompt: expected string or chat message array.`); + } + const kind = Array.isArray(promptValue) ? 'chat' : 'string'; + return { + identity: { + id: explicitId ?? `${kind}-${stablePromptId(promptValue)}`, + ...(label ? { label } : {}), + kind, + }, + input: promptValue, + }; + } + + if (rawPrompt.raw !== undefined) { + const rawValue = asString(rawPrompt.raw); + if (!rawValue) { + throw new Error(`Invalid prompts[${index}].raw: expected non-empty string.`); + } + return { + identity: { + id: explicitId ?? `string-${stablePromptId(rawValue)}`, + ...(label ? { label } : {}), + kind: 'string', + }, + input: rawValue, + }; + } + + if (isTestMessage(rawPrompt)) { + return { + identity: { + id: explicitId ?? `chat-${stablePromptId(rawPrompt)}`, + ...(label ? { label } : {}), + kind: 'chat', + }, + input: [rawPrompt], + }; + } + + throw new Error(`Invalid prompts[${index}]: expected prompt, messages, file, or function.`); +} + +async function parseSuitePrompts( + rawPrompts: JsonValue | undefined, + searchRoots: readonly string[], +): Promise { + if (rawPrompts === undefined || rawPrompts === null) { + return undefined; + } + + const entries = + Array.isArray(rawPrompts) && !isChatPromptArray(rawPrompts) ? rawPrompts : [rawPrompts]; + const prompts: PromptDefinition[] = []; + for (let index = 0; index < entries.length; index++) { + prompts.push(await parsePromptDefinition(entries[index] as JsonValue, searchRoots, index)); + } + return prompts; +} + +function renderPromptInput(prompt: PromptDefinition, vars: JsonObject | undefined): JsonValue { + return interpolateCaseField(prompt.input, vars); +} + +function expandPromptMatrix( + rawCases: readonly JsonValue[], + prompts: readonly PromptDefinition[] | undefined, + suite: RawTestSuite, +): PromptExpansionResult { + const promptById = new Map(); + const sourceTestIdById = new Map(); + + if (!prompts) { + if (suite.input !== undefined || suite.input_files !== undefined) { + logWarning( + "Top-level 'input' and 'input_files' are deprecated. Use top-level 'prompts' plus tests[].vars instead.", + ); + } else if ( + rawCases.some( + (rawCase) => + isJsonObject(rawCase) && + (rawCase.input !== undefined || rawCase.input_files !== undefined), + ) + ) { + logWarning("tests[].input is deprecated. Use top-level 'prompts' plus tests[].vars instead."); + } + return { rawCases, promptById, sourceTestIdById }; + } + + if (suite.input !== undefined || suite.input_files !== undefined) { + throw new Error("Top-level 'input' and 'input_files' cannot be combined with 'prompts'."); + } + + const expandedCases: JsonValue[] = []; + for (const rawCase of rawCases) { + if (!isJsonObject(rawCase)) { + expandedCases.push(rawCase); + continue; + } + if (rawCase.input !== undefined || rawCase.input_files !== undefined) { + throw new Error( + "tests[].input and tests[].input_files have been removed from the preferred prompt contract. Use top-level 'prompts' plus tests[].vars.", + ); + } + + const sourceTestId = asString(rawCase.id); + const vars = isJsonObject(rawCase.vars) ? rawCase.vars : undefined; + for (const prompt of prompts) { + const promptId = safePromptId(prompt.identity.id); + const expandedId = + sourceTestId && prompts.length > 1 ? `${sourceTestId}__prompt_${promptId}` : sourceTestId; + const expandedDependsOn = Array.isArray(rawCase.depends_on) + ? rawCase.depends_on.map((dep) => + typeof dep === 'string' && prompts.length > 1 ? `${dep}__prompt_${promptId}` : dep, + ) + : rawCase.depends_on; + const expandedCase: JsonObject = { + ...rawCase, + ...(expandedId ? { id: expandedId } : {}), + ...(expandedDependsOn !== undefined ? { depends_on: expandedDependsOn } : {}), + input: renderPromptInput(prompt, vars), + }; + expandedCases.push(expandedCase); + if (expandedId) { + promptById.set(expandedId, prompt.identity); + if (sourceTestId) { + sourceTestIdById.set(expandedId, sourceTestId); + } + } + } + } + + return { rawCases: expandedCases, promptById, sourceTestIdById }; +} + async function loadNunjucksFilters( rawFilters: JsonValue | undefined, evalFileDir: string, @@ -671,6 +1087,10 @@ async function loadTestsFromParsedYamlValue( throw new Error(`Invalid test file format: ${evalFilePath} - missing 'tests' field`); } + const promptDefinitions = await parseSuitePrompts(suite.prompts, searchRoots); + const promptExpansion = expandPromptMatrix(expandedTestCases, promptDefinitions, suite); + expandedTestCases = promptExpansion.rawCases; + const suiteWorkspace = await resolveWorkspaceConfig(suite.workspace, evalFileDir); const rawSuiteInput = suite.input; @@ -700,6 +1120,8 @@ async function loadTestsFromParsedYamlValue( const caseVars = isJsonObject(testCaseConfig.vars) ? testCaseConfig.vars : undefined; const renderedCase = interpolateRawEvalCase(testCaseConfig, caseVars, nunjucksFilters); const id = asString(renderedCase.id); + const promptIdentity = id ? promptExpansion.promptById.get(id) : undefined; + const sourceTestId = id ? promptExpansion.sourceTestIdById.get(id) : undefined; // Skip tests that don't match the filter pattern (glob supported) if (filterPattern && (!id || !matchesFilter(id, filterPattern))) { @@ -953,9 +1375,11 @@ async function loadTestsFromParsedYamlValue( const testCase: EvalTest = { id, + ...(sourceTestId ? { testId: sourceTestId } : {}), suite: suiteName, category, conversation_id: conversationId, + ...(promptIdentity ? { prompt: promptIdentity } : {}), question: question, input: inputMessages, expected_output: outputSegments, diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 52312c0a8..6c50e131b 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -90,6 +90,150 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.experimentConfig?.threshold).toBe(0.9); }); + it('expands top-level prompts across tests with per-test vars', async () => { + const evalPath = path.join(tempDir, 'prompt-matrix.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: prompt-matrix-suite', + 'prompts:', + ' - id: direct', + ' label: Direct', + ' prompt: "Summarize {{ topic }}."', + ' - id: terse', + ' label: Terse', + ' prompt: "In one sentence, summarize {{ topic }}."', + 'targets:', + ' - id: openai:gpt-5.4-mini', + ' label: mini', + ' - id: local-codex', + 'tests:', + ' - id: docs', + ' vars:', + ' topic: release notes', + ' expected_output: concise release-note summary', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.tests.map((test) => test.id)).toEqual([ + 'docs__prompt_direct', + 'docs__prompt_terse', + ]); + expect(suite.tests.map((test) => test.testId)).toEqual(['docs', 'docs']); + expect(suite.tests.map((test) => test.prompt)).toEqual([ + { id: 'direct', label: 'Direct', kind: 'string' }, + { id: 'terse', label: 'Terse', kind: 'string' }, + ]); + expect(suite.tests.map((test) => test.question)).toEqual([ + 'Summarize release notes.', + 'In one sentence, summarize release notes.', + ]); + expect(suite.targets).toEqual(['openai:gpt-5.4-mini', 'local-codex']); + expect(suite.targetRefs).toEqual([ + { name: 'openai:gpt-5.4-mini', id: 'openai:gpt-5.4-mini', label: 'mini' }, + { name: 'local-codex', id: 'local-codex' }, + ]); + }); + + it('loads function prompt sources from top-level prompts', async () => { + const promptScriptPath = path.join(tempDir, 'prompt-source.js'); + const evalPath = path.join(tempDir, 'function-prompts.eval.yaml'); + await writeFile( + promptScriptPath, + "console.log(JSON.stringify({ prompt: 'Explain {{ topic }} with one concrete example.' }));\n", + ); + await writeFile( + evalPath, + [ + 'name: function-prompt-suite', + 'prompts:', + ' - id: generated', + ' label: Generated', + ' function_file: prompt-source.js', + 'tests:', + ' - id: docs', + ' vars:', + ' topic: release notes', + ' expected_output: concrete release-note explanation', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.tests).toHaveLength(1); + expect(suite.tests[0]?.id).toBe('docs'); + expect(suite.tests[0]?.testId).toBe('docs'); + expect(suite.tests[0]?.prompt).toEqual({ + id: 'generated', + label: 'Generated', + kind: 'function', + }); + expect(suite.tests[0]?.question).toBe('Explain release notes with one concrete example.'); + }); + + it('loads chat and file prompts from the top-level prompt matrix', async () => { + const promptPath = path.join(tempDir, 'prompt.md'); + const evalPath = path.join(tempDir, 'prompt-sources.eval.yaml'); + await writeFile(promptPath, 'Review {{ file_name }}.\n'); + await writeFile( + evalPath, + [ + 'name: prompt-sources-suite', + 'prompts:', + ' - id: chat', + ' messages:', + ' - role: system', + ' content: Be precise.', + ' - role: user', + ' content: "Inspect {{ file_name }}."', + ' - id: file', + ' file: prompt.md', + 'tests:', + ' - id: inspect', + ' vars:', + ' file_name: README.md', + ' criteria: useful', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.tests).toHaveLength(2); + expect(suite.tests[0]?.input).toEqual([ + { role: 'system', content: 'Be precise.' }, + { role: 'user', content: 'Inspect README.md.' }, + ]); + expect(suite.tests[1]?.question).toBe('Review README.md.'); + expect(suite.tests[1]?.prompt).toEqual({ + id: 'file', + label: 'prompt.md', + kind: 'file', + }); + }); + + it('rejects tests input when top-level prompts are authored', async () => { + const evalPath = path.join(tempDir, 'mixed-prompt-contract.eval.yaml'); + await writeFile( + evalPath, + [ + 'prompts:', + ' - hello', + 'tests:', + ' - id: one', + ' input: legacy', + ' criteria: ok', + '', + ].join('\n'), + ); + + await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/tests\[\]\.input/); + }); + it('parses evaluate_options.budget_usd and prefers it over legacy top-level budget_usd', async () => { const evalPath = path.join(tempDir, 'evaluate-options-budget.eval.yaml'); await writeFile( diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index c6a0ae034..7513d842a 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -622,6 +622,8 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { { name: 'registry-agent' }, { name: 'inline-agent', + id: 'mock', + label: 'inline-agent', definition: expect.objectContaining({ id: 'mock', name: 'inline-agent', diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 4acf92dd1..bc33f4618 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -3181,6 +3181,46 @@ describe('suite-level total budget guardrail', () => { expect(results[2].error).toContain('Run budget exceeded'); expect(results[3].error).toContain('Run budget exceeded'); }); + + it('preserves authored prompt identity on budget-skipped prompt-expanded rows', async () => { + const prompt = { id: 'direct', label: 'Direct prompt', kind: 'string' as const }; + const provider: Provider = { + id: 'budget:mock', + kind: 'mock' as const, + targetName: 'mock', + async invoke(): Promise { + return { + output: [{ role: 'assistant', content: 'response' }], + costUsd: 3.0, + }; + }, + }; + + const evalCases: EvalTest[] = [ + { ...baseTestCase, id: 'warmup' }, + { + ...baseTestCase, + id: 'docs__prompt_direct', + testId: 'docs', + prompt, + }, + ]; + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: baseTarget, + providerFactory: () => provider, + evaluators: evaluatorRegistry, + evalCases, + maxConcurrency: 1, + runBudgetTracker: new RunBudgetTracker(2.0), + }); + + expect(results[1]?.budgetExceeded).toBe(true); + expect(results[1]?.testId).toBe('docs'); + expect(results[1]?.prompt).toEqual(prompt); + }); }); describe('fail_on_error tolerance', () => { @@ -3223,10 +3263,51 @@ describe('fail_on_error tolerance', () => { // Remaining cases should be halted by error_threshold_exceeded expect(results[1].executionStatus).toBe('execution_error'); expect(results[1].failureReasonCode).toBe('error_threshold_exceeded'); + expect(results[1].testId).toBe('skip-case-1'); expect(results[2].executionStatus).toBe('execution_error'); expect(results[2].failureReasonCode).toBe('error_threshold_exceeded'); }); + it('preserves authored prompt identity on fail_on_error halted prompt-expanded rows', async () => { + let callCount = 0; + const prompt = { id: 'direct', label: 'Direct prompt', kind: 'string' as const }; + const errorOnFirstProvider: Provider = { + id: 'mock:error-on-first', + kind: 'mock' as const, + targetName: 'error-on-first', + async invoke(): Promise { + callCount++; + if (callCount === 1) { + throw new Error('Provider failed'); + } + return { output: [{ role: 'assistant', content: 'ok' }] }; + }, + }; + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: baseTarget, + providerFactory: () => errorOnFirstProvider, + evaluators: evaluatorRegistry, + evalCases: [ + { ...baseTestCase, id: 'fail-case' }, + { + ...baseTestCase, + id: 'docs__prompt_direct', + testId: 'docs', + prompt, + }, + ], + failOnError: true, + maxConcurrency: 1, + }); + + expect(results[1]?.failureReasonCode).toBe('error_threshold_exceeded'); + expect(results[1]?.testId).toBe('docs'); + expect(results[1]?.prompt).toEqual(prompt); + }); + it('fail_on_error: false never halts on errors', async () => { let callCount = 0; const alwaysErrorProvider: Provider = { diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 7f0511f1c..6a8eee8b8 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -196,6 +196,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -257,6 +263,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -969,6 +981,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1030,6 +1048,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1295,6 +1319,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1356,6 +1386,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1621,6 +1657,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1682,6 +1724,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1916,6 +1964,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -1977,6 +2031,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -2437,6 +2497,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -3806,6 +3872,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -5696,6 +5768,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -5757,6 +5835,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -6022,6 +6106,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -6083,6 +6173,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -6348,6 +6444,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -6409,6 +6511,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -6643,6 +6751,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -6704,6 +6818,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -7164,6 +7284,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -8533,6 +8659,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -10399,6 +10531,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -10460,6 +10598,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -10725,6 +10869,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -10786,6 +10936,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -11051,6 +11207,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -11112,6 +11274,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -11535,6 +11703,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -11596,6 +11770,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -11861,6 +12041,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -11922,6 +12108,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -12187,6 +12379,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -12248,6 +12446,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -12482,6 +12686,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -12543,6 +12753,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -12829,6 +13045,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -12890,6 +13112,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -13155,6 +13383,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -13216,6 +13450,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -13481,6 +13721,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -13542,6 +13788,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -13776,6 +14028,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -13837,6 +14095,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -14058,6 +14322,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -14119,6 +14389,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -14384,6 +14660,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -14445,6 +14727,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -14710,6 +14998,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -14771,6 +15065,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -15005,6 +15305,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -15066,6 +15372,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -15526,6 +15838,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" }, @@ -16895,6 +17213,12 @@ "raw": { "type": "string" }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, "path": { "type": "string" },