From 97139bcdb6a939126d354ed7c31f8906f751252b Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Sat, 18 Jul 2026 11:07:15 -0400 Subject: [PATCH] Expose visual explanation controls --- package.json | 1 + packages/runtime-core/src/command-registry.ts | 5 +- .../src/browser-visual-compare.ts | 31 +++- tests/browser-visual-compare-contract.test.ts | 134 ++++++++++++++++++ 4 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 tests/browser-visual-compare-contract.test.ts diff --git a/package.json b/package.json index 3a229d52..1daf6c3a 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "test:browser-visual-compare-fair-ratio": "tsx tests/browser-visual-compare-fair-ratio.test.ts", "test:browser-visual-compare-layout-drift": "tsx tests/browser-visual-compare-layout-drift.test.ts", "test:browser-visual-compare-capture-reliability": "tsx tests/browser-visual-compare-capture-reliability.test.ts", + "test:browser-visual-compare-contract": "npm run build && tsx tests/browser-visual-compare-contract.test.ts", "test:browser-visual-compare-dom-snapshots": "npm run build && tsx tests/browser-visual-compare-dom-snapshots.test.ts", "test:browser-session-public-dto": "tsx tests/browser-session-public-dto.test.ts", "test:browser-playground-session-run": "tsx tests/browser-playground-session-run.test.ts", diff --git a/packages/runtime-core/src/command-registry.ts b/packages/runtime-core/src/command-registry.ts index 7b54d840..c4d68245 100644 --- a/packages/runtime-core/src/command-registry.ts +++ b/packages/runtime-core/src/command-registry.ts @@ -1217,7 +1217,7 @@ export const commandRegistry = [ acceptedArgs: [ { name: "source-url", description: "Source browser target path or absolute URL.", format: "path or URL" }, { name: "candidate-url", description: "Candidate browser target path or absolute URL.", format: "path or URL" }, - { name: "matrix-json", description: "Optional comparison matrix object with comparisons and optional viewports arrays; each comparison may provide source/candidate targets, labels, viewport, and wait settings.", format: "JSON object" }, + { name: "matrix-json", description: "Optional comparison matrix object with comparisons and optional viewports arrays; each comparison may provide source/candidate targets, labels, viewport, wait settings, maxExplanationElements/maxExplanationCandidates/explainSelectors, or their kebab-case max-explanation-elements/max-explanation-candidates/explain-selector aliases.", format: "JSON object" }, { name: "source-screenshot", description: "Existing source PNG screenshot path on the host.", format: "path" }, { name: "candidate-screenshot", description: "Existing candidate PNG screenshot path on the host.", format: "path" }, { name: "source-dom-snapshot", description: "Optional source DOM/style sidecar snapshot path for screenshot-backed visual explanations.", format: "path" }, @@ -1232,6 +1232,9 @@ export const commandRegistry = [ { name: "threshold", description: "Pixelmatch color threshold; defaults to 0.1.", format: "number between 0 and 1" }, { name: "include-aa", description: "Include anti-aliased pixels in mismatch count; defaults to false.", format: "boolean" }, { name: "max-regions", description: "Maximum mismatch regions to report; defaults to 8.", format: "positive integer" }, + { name: "max-explanation-elements", description: "Maximum DOM elements included in visual explanation attribution; defaults to 25.", format: "positive integer" }, + { name: "max-explanation-candidates", description: "Maximum visible DOM elements captured from each target before visual explanation attribution; defaults to 160.", format: "positive integer" }, + { name: "explain-selector", description: "Targeted selector included in visual explanation attribution. Supply once per selector.", repeatable: true, format: "CSS selector" }, ], outputShape: "wp-codebox/visual-compare/v1 JSON summary plus files/browser/visual-compare/source.png, candidate.png, diff.png, visual-diff.json, visual-explanation.json when DOM/style context is available, and optional baseline delta evidence when a previous visual comparison artifact is supplied. Matrix runs emit wp-codebox/visual-compare-matrix/v1 in files/browser/visual-compare/matrix-summary.json plus per-comparison subdirectories.", outputSchema: objectEnvelopeSchema("wp-codebox/visual-compare/v1", { diff --git a/packages/runtime-playground/src/browser-visual-compare.ts b/packages/runtime-playground/src/browser-visual-compare.ts index 64a315c5..ec4886f4 100644 --- a/packages/runtime-playground/src/browser-visual-compare.ts +++ b/packages/runtime-playground/src/browser-visual-compare.ts @@ -1233,6 +1233,7 @@ function visualCompareMatrixEndpoint(args: string[], role: "source" | "candidate function visualCompareMatrixOptions(args: string[]): Record { const requestedViewport = viewportArg(args, "viewport") + const explainSelectors = visualCompareExplainSelectors(args) return { waitFor: argValue(args, "wait-for")?.trim() || "domcontentloaded", durationMs: durationArg(args, "duration", 0), @@ -1243,6 +1244,7 @@ function visualCompareMatrixOptions(args: string[]): Record { maxRegions: positiveIntegerArg(args, "max-regions", 8), maxExplanationElements: positiveIntegerArg(args, "max-explanation-elements", 25), maxExplanationCandidates: positiveIntegerArg(args, "max-explanation-candidates", 160), + ...(explainSelectors.length > 0 ? { explainSelectors } : {}), } } @@ -1420,10 +1422,31 @@ function visualCompareMatrixArgs(record: Record): string[] { ["max-explanation-elements", ["max-explanation-elements", "maxExplanationElements"]], ["max-explanation-candidates", ["max-explanation-candidates", "maxExplanationCandidates"]], ] - return fields.flatMap(([argName, keys]) => { + const args = fields.flatMap(([argName, keys]) => { const value = visualCompareMatrixValue(record, keys) return value === undefined ? [] : [`${argName}=${String(value)}`] }) + const selectors = visualCompareMatrixSelectors(record) + return [...args, ...selectors.map((selector) => `explain-selector=${selector}`)] +} + +function visualCompareMatrixSelectors(record: Record): string[] { + const raw = visualCompareMatrixValue(record, ["explain-selector", "explainSelector", "explain-selectors", "explainSelectors"]) + if (raw === undefined) { + return [] + } + const values = Array.isArray(raw) ? raw : [raw] + const selectors = new Set() + for (const value of values) { + if (typeof value !== "string") { + throw new Error("matrix-json explainSelectors must be a string or an array of strings") + } + const selector = value.trim() + if (selector) { + selectors.add(selector) + } + } + return [...selectors] } function visualCompareMatrixString(record: Record, keys: string[]): string | undefined { @@ -1445,7 +1468,7 @@ function sanitizeVisualCompareMatrixName(name: string): string { } function mergeVisualCompareMatrixArgs(baseArgs: string[], entryArgs: string[]): string[] { - const merged = baseArgs.filter((arg) => !entryArgs.some((entryArg) => arg.slice(0, arg.indexOf("=") + 1) === entryArg.slice(0, entryArg.indexOf("=") + 1))) + const merged = baseArgs.filter((arg) => arg.startsWith("explain-selector=") || !entryArgs.some((entryArg) => arg.slice(0, arg.indexOf("=") + 1) === entryArg.slice(0, entryArg.indexOf("=") + 1))) merged.push(...entryArgs) return merged } @@ -2914,9 +2937,9 @@ function positiveIntegerArg(args: string[], name: string, fallback: number): num if (!raw) { return fallback } - const parsed = Number.parseInt(raw, 10) - if (!Number.isFinite(parsed) || parsed <= 0) { + if (!/^[1-9]\d*$/.test(raw)) { throw new Error(`${name} must be a positive integer`) } + const parsed = Number.parseInt(raw, 10) return parsed } diff --git a/tests/browser-visual-compare-contract.test.ts b/tests/browser-visual-compare-contract.test.ts new file mode 100644 index 00000000..a77e82b3 --- /dev/null +++ b/tests/browser-visual-compare-contract.test.ts @@ -0,0 +1,134 @@ +import assert from "node:assert/strict" +import { readFile, writeFile } from "node:fs/promises" +import { join } from "node:path" + +import { PNG } from "pngjs" +import { commandRegistry } from "../packages/runtime-core/src/command-registry.js" +import { runVisualCompareCommand } from "../packages/runtime-playground/dist/browser-visual-compare.js" +import { withTempDir } from "../scripts/test-kit.js" + +const visualCompare = commandRegistry.find((definition) => definition.id === "wordpress.visual-compare") +assert.ok(visualCompare, "wordpress.visual-compare is registered") + +const acceptedArgs = visualCompare.acceptedArgs +const maxElements = acceptedArgs.find((arg) => arg.name === "max-explanation-elements") +const maxCandidates = acceptedArgs.find((arg) => arg.name === "max-explanation-candidates") +const selector = acceptedArgs.find((arg) => arg.name === "explain-selector") +assert.deepEqual(maxElements && { format: maxElements.format }, { format: "positive integer" }) +assert.deepEqual(maxCandidates && { format: maxCandidates.format }, { format: "positive integer" }) +assert.deepEqual(selector && { repeatable: selector.repeatable, format: selector.format }, { repeatable: true, format: "CSS selector" }) +const matrixDescription = acceptedArgs.find((arg) => arg.name === "matrix-json")?.description ?? "" +for (const field of ["maxExplanationElements", "maxExplanationCandidates", "explainSelectors", "max-explanation-elements", "max-explanation-candidates", "explain-selector"]) { + assert.match(matrixDescription, new RegExp(field)) +} + +async function writePng(path: string): Promise { + const png = new PNG({ width: 1, height: 1 }) + png.data.set([255, 255, 255, 255]) + await writeFile(path, PNG.sync.write(png)) +} + +async function visualCompareRun(artifactRoot: string, args: string[]) { + return runVisualCompareCommand({ + artifactRoot, + server: { + serverUrl: "http://127.0.0.1:1", + playground: { run: async () => ({ text: "" }) }, + async [Symbol.asyncDispose]() {}, + }, + spec: { command: "wordpress.visual-compare", args }, + }) +} + +const expectedOptions = (maxExplanationElements: number, maxExplanationCandidates: number, explainSelectors?: string[]) => ({ + waitFor: "domcontentloaded", + durationMs: 0, + timeoutMs: 120_000, + fullPage: true, + maxFullPageHeight: 20_000, + threshold: 0.1, + includeAA: false, + maxRegions: 8, + maxExplanationElements, + maxExplanationCandidates, + ...(explainSelectors ? { explainSelectors } : {}), +}) + +await withTempDir("wp-codebox-visual-compare-contract-", async (artifactRoot) => { + const sourceScreenshot = join(artifactRoot, "source.png") + const candidateScreenshot = join(artifactRoot, "candidate.png") + await Promise.all([writePng(sourceScreenshot), writePng(candidateScreenshot)]) + + const defaults = JSON.parse((await visualCompareRun(artifactRoot, [`source-screenshot=${sourceScreenshot}`, `candidate-screenshot=${candidateScreenshot}`])).output) + assert.deepEqual(defaults.options, expectedOptions(25, 160)) + + const pair = JSON.parse((await visualCompareRun(artifactRoot, [ + `source-screenshot=${sourceScreenshot}`, + `candidate-screenshot=${candidateScreenshot}`, + "max-explanation-elements=40", + "max-explanation-candidates=240", + "explain-selector=main", + "explain-selector=body", + ])).output) + assert.deepEqual(pair.options, expectedOptions(40, 240, ["main", "body"])) + assert.equal(pair.schema, "wp-codebox/visual-compare/v1") + assert.equal(pair.command, "wordpress.visual-compare") + assert.equal(pair.status, "identical") + assert.deepEqual(pair.files, { + sourceScreenshot: "files/browser/visual-compare/source.png", + candidateScreenshot: "files/browser/visual-compare/candidate.png", + diffScreenshot: "files/browser/visual-compare/diff.png", + visualDiff: "files/browser/visual-compare/visual-diff.json", + blocksEngineVisualParity: "files/browser/visual-compare/blocks-engine-visual-parity-report.json", + summary: "files/browser/visual-compare/summary.json", + }) + assert.deepEqual(pair.comparison, { + source: { width: 1, height: 1 }, + candidate: { width: 1, height: 1 }, + diff: { width: 1, height: 1 }, + mismatchPixels: 0, + totalPixels: 1, + mismatchRatio: 0, + overlapMismatchPixels: 0, + overlapPixels: 1, + overlapMismatchRatio: 0, + dimensionMismatch: false, + dimensionDeltaPixels: 0, + dimensionDeltaRatio: 0, + regions: [], + }) + for (const hash of Object.values(pair.hashes) as Array<{ algorithm: string; value: string }>) { + assert.equal(hash.algorithm, "sha256") + assert.match(hash.value, /^[a-f0-9]{64}$/) + } + const persistedPair = JSON.parse(await readFile(join(artifactRoot, pair.files.summary), "utf8")) + assert.deepEqual(persistedPair.options, pair.options) + assert.deepEqual(persistedPair.files, pair.files) + + for (const [arg, message] of [["max-explanation-elements=0", "max-explanation-elements"], ["max-explanation-candidates=0", "max-explanation-candidates"], ["max-explanation-elements=1.5", "max-explanation-elements"], ["max-explanation-candidates=160px", "max-explanation-candidates"]]) { + await assert.rejects(visualCompareRun(artifactRoot, [`source-screenshot=${sourceScreenshot}`, `candidate-screenshot=${candidateScreenshot}`, arg]), new RegExp(`${message} must be a positive integer`)) + } + + const matrix = JSON.parse((await visualCompareRun(artifactRoot, [ + "explain-selector=main", + "explain-selector=body", + `matrix-json=${JSON.stringify({ comparisons: [{ name: "camel-case", sourceScreenshot, candidateScreenshot, maxExplanationElements: 50, maxExplanationCandidates: 300, explainSelectors: ["body", "article"] }, { name: "kebab-case", "source-screenshot": sourceScreenshot, "candidate-screenshot": candidateScreenshot, "max-explanation-elements": 60, "max-explanation-candidates": 320, "explain-selector": "article" }] })}`, + ])).output) + assert.equal(matrix.schema, "wp-codebox/visual-compare-matrix/v1") + assert.equal(matrix.command, "wordpress.visual-compare") + assert.equal(matrix.complete, true) + assert.deepEqual(matrix.metrics, { expectedComparisons: 2, comparisons: 2, missing: 0, failed: 0, identical: 2, different: 0, maxMismatchRatio: 0, meanMismatchRatio: 0, maxOverlapMismatchRatio: 0, meanOverlapMismatchRatio: 0, maxMismatchPixels: 0, meanMismatchPixels: 0 }) + assert.deepEqual(matrix.comparisons[0].options, expectedOptions(50, 300, ["main", "body", "article"])) + assert.deepEqual(matrix.comparisons[1].options, expectedOptions(60, 320, ["main", "body", "article"])) + const persistedMatrix = JSON.parse(await readFile(join(artifactRoot, matrix.files.summary), "utf8")) + assert.deepEqual(persistedMatrix.comparisons.map((comparison: { options: unknown }) => comparison.options), matrix.comparisons.map((comparison: { options: unknown }) => comparison.options)) + + for (const [field, value] of [["maxExplanationElements", "1.5"], ["maxExplanationCandidates", 0]]) { + await assert.rejects( + visualCompareRun(artifactRoot, [`matrix-json=${JSON.stringify({ comparisons: [{ sourceScreenshot, candidateScreenshot, [field]: value }] })}`]), + new RegExp(`${field === "maxExplanationElements" ? "max-explanation-elements" : "max-explanation-candidates"} must be a positive integer`), + ) + } +}) + +console.log("browser visual compare contract passed")