diff --git a/AGENTS.md b/AGENTS.md
index 8f0db60..da5c878 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -49,6 +49,20 @@ rules below are the parts that were worth keeping.
enforce a path convention, so update the explicit mapping when a suite is added or its doc moves.
Groups without a matching explainer render no docs link. Do not guess a path or link to an
unrelated general guide.
+- **Every group kind has a summary, by default.** [`web/lib/summary.ts`](web/lib/summary.ts)
+ has no allowlist gate: a suite that lands in one of the five fact tables gets a rollup card
+ from its first ingest. The three timing families (query, random access, vector search) rank
+ through one `rankSeries` model, so a new suite in any of them needs no summary code at all.
+ `collectGroupSummary`'s exhaustive switch makes a missing arm for a sixth fact table a compile
+ error rather than a silently blank card. Do NOT reintroduce a per-dataset allowlist; the v2-era
+ one is what left `spatialbench`, `fineweb`, `gharchive`, `appian`, `public-bi`,
+ `clickbench-sorted`, and every vector-search group with no card.
+- **A summary ranks the whole group, never one chart.** Random access is the cautionary case:
+ the producer emits `dataset` as `{dataset}/{pattern}`, so the group holds ~nine charts, and the
+ old summary published the alphabetically first chart's raw times under the group-wide title
+ "Random Access Performance" — reporting `lance` at 352us when it is over 1ms on most of the
+ other charts. Rank across every bucket in the group, impute the missing-series penalty where a
+ series skipped one, and report `measured`/`total` so a partially covered series is legible.
- **Don't write a server-side classifier for live ingest.** The emitter produces structured
records directly. Classifying loose name strings at read time was the v2-era weakness every
later generation existed to escape; it belongs nowhere in the live pipeline.
diff --git a/web/components/GroupSection.test.tsx b/web/components/GroupSection.test.tsx
index cd5c7de..6bacd8c 100644
--- a/web/components/GroupSection.test.tsx
+++ b/web/components/GroupSection.test.tsx
@@ -23,7 +23,7 @@ const RANDOM_ACCESS: Group = {
summary: {
type: 'randomAccess',
title: 'Random Access Performance',
- rankings: [{ name: 'vortex', time: 1_500_000, ratio: 1 }],
+ rankings: [{ name: 'vortex', score: 1, totalRuntime: 1_500_000, measured: 1, total: 1 }],
explanation: 'lower is better',
},
description: 'Tests selecting arbitrary row indices on NVMe',
diff --git a/web/components/SummaryCard.test.tsx b/web/components/SummaryCard.test.tsx
index a91e1fd..f0b4d82 100644
--- a/web/components/SummaryCard.test.tsx
+++ b/web/components/SummaryCard.test.tsx
@@ -16,15 +16,15 @@ describe('SummaryCard', () => {
expect(render(undefined)).toBe('');
});
- it('renders a randomAccess card with ranks, ns times, and ratios', () => {
+ it('renders a randomAccess card with ranks, scores, and total runtimes', () => {
const html = render({
type: 'randomAccess',
title: 'Random Access Performance',
rankings: [
- { name: 'vortex', time: 1_500_000, ratio: 1 },
- { name: 'parquet', time: 3_000_000, ratio: 2 },
+ { name: 'vortex', score: 1, totalRuntime: 1_500_000, measured: 2, total: 2 },
+ { name: 'parquet', score: 2, totalRuntime: 3_000_000, measured: 2, total: 2 },
],
- explanation: 'Random access time | Ratio to fastest (lower is better)',
+ explanation: 'Geomean of take time ratio to fastest across every chart (lower is better)',
});
expect(html).toContain('class="benchmark-scores-summary"');
expect(html).toContain('Random Access Performance
');
@@ -36,7 +36,25 @@ describe('SummaryCard', () => {
expect(html).toContain('parquet');
expect(html).toContain('3.00 ms');
expect(html).toContain('2.00x');
- expect(html).toContain('Random access time | Ratio to fastest (lower is better)');
+ expect(html).toContain(
+ 'Geomean of take time ratio to fastest across every chart (lower is better)',
+ );
+ });
+
+ it('flags a partially measured series in its hover text', () => {
+ const html = render({
+ type: 'randomAccess',
+ title: 'Random Access Performance',
+ rankings: [
+ { name: 'vortex', score: 1, totalRuntime: 1_500_000, measured: 9, total: 9 },
+ { name: 'lance', score: 3, totalRuntime: 3_000_000, measured: 4, total: 9 },
+ ],
+ explanation: 'e',
+ });
+ // The full-coverage series keeps a bare label; the partial one says so, so
+ // a penalty-inflated score is never presented as a like-for-like number.
+ expect(html).toContain('title="vortex"');
+ expect(html).toContain('measured in 4 of 9 charts');
});
it('renders nothing for a randomAccess card with no rankings', () => {
@@ -96,7 +114,15 @@ describe('SummaryCard', () => {
const html = render({
type: 'queryBenchmark',
title: 'Performance Summary',
- rankings: [{ name: 'vortex:vortex-file', score: 1.0, totalRuntime: 5_000_000_000 }],
+ rankings: [
+ {
+ name: 'vortex:vortex-file',
+ score: 1.0,
+ totalRuntime: 5_000_000_000,
+ measured: 1,
+ total: 1,
+ },
+ ],
explanation: 'lower is better',
});
expect(html).toContain('#1');
@@ -104,4 +130,18 @@ describe('SummaryCard', () => {
expect(html).toContain('1.00x');
expect(html).toContain('5.00 s');
});
+
+ it('renders a vectorSearch card through the shared timing arm', () => {
+ const html = render({
+ type: 'vectorSearch',
+ title: 'Vector Search Performance',
+ rankings: [
+ { name: 'vortex-turboquant', score: 1.0, totalRuntime: 7_000, measured: 2, total: 2 },
+ ],
+ explanation: 'lower is better',
+ });
+ expect(html).toContain('Vector Search Performance
');
+ expect(html).toContain('vortex-turboquant');
+ expect(html).toContain('1.00x');
+ });
});
diff --git a/web/components/SummaryCard.tsx b/web/components/SummaryCard.tsx
index 3cd63c9..6d57dc3 100644
--- a/web/components/SummaryCard.tsx
+++ b/web/components/SummaryCard.tsx
@@ -3,7 +3,20 @@
import { displayFormat, displaySeriesLabel } from '@/lib/chart-format';
import { formatTimeNs } from '@/lib/format';
-import type { Summary } from '@/lib/summary';
+import type { SeriesRanking, Summary } from '@/lib/summary';
+
+/**
+ * Hover text for a ranked series: its full label plus, when the series was not
+ * measured everywhere, how much of its score came from the missing-bucket
+ * penalty. A partially measured series is ranked, not hidden, so the coverage
+ * has to be legible somewhere.
+ */
+function seriesTitle(item: SeriesRanking): string {
+ const label = displaySeriesLabel(item.name);
+ return item.measured >= item.total
+ ? label
+ : `${label} - measured in ${item.measured} of ${item.total} charts; the rest scored by the missing-series penalty`;
+}
/**
* The per-group summary card.
@@ -11,7 +24,8 @@ import type { Summary } from '@/lib/summary';
* Every [`Summary`] variant renders the same `.benchmark-scores-summary` shape
* (a `.scores-title`, a `.scores-list` of `.score-item` rows, and a
* `.scores-explanation` footer); only the rank label, value, and optional
- * runtime change. The card stays visible whether or not the enclosing group is
+ * runtime change. The three timing families (query, random access, vector
+ * search) share one arm because they share one [`SeriesRanking`] shape. The card stays visible whether or not the enclosing group is
* expanded (the CSS only hides `.chart-grid` when the disclosure is closed), so
* the at-a-glance rankings show without expanding the group.
*
@@ -22,30 +36,6 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
return null;
}
switch (summary.type) {
- case 'randomAccess':
- if (summary.rankings.length === 0) {
- return null;
- }
- return (
-
- {summary.title}
-
- {summary.rankings.map((item, idx) => (
-
- #{idx + 1}
-
- {displaySeriesLabel(item.name)}
-
-
- {formatTimeNs(item.time)}
- {item.ratio.toFixed(2)}x
-
-
- ))}
-
{summary.explanation}
-
- );
case 'compression': {
if (summary.rankings.length === 0) {
return null;
@@ -118,6 +108,8 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
);
case 'queryBenchmark':
+ case 'randomAccess':
+ case 'vectorSearch':
if (summary.rankings.length === 0) {
return null;
}
@@ -128,7 +120,7 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
{summary.rankings.map((item, idx) => (
#{idx + 1}
-
+
{displaySeriesLabel(item.name)}
diff --git a/web/lib/data-cache.test.ts b/web/lib/data-cache.test.ts
index 4cf46f0..4de1c1c 100644
--- a/web/lib/data-cache.test.ts
+++ b/web/lib/data-cache.test.ts
@@ -63,9 +63,9 @@ describe('data-cache wrappers', () => {
expect(BENCH_DATA_TAG).toBe('bench-data');
expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([
- 'data-cache:group-charts:v4:n100',
+ 'data-cache:group-charts:v5:n100',
]);
- expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v4']);
+ expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v5']);
for (const call of cacheCalls) {
expect(call.options.tags).toEqual([BENCH_DATA_TAG]);
expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS);
diff --git a/web/lib/data-cache.ts b/web/lib/data-cache.ts
index fda1b92..f5a83fb 100644
--- a/web/lib/data-cache.ts
+++ b/web/lib/data-cache.ts
@@ -46,7 +46,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_
* This value must change when a deployment cannot read the preceding shape.
* It is independent of the producer-facing benchmark schema version.
*/
-const GROUP_PAYLOAD_CACHE_VERSION = 'v4';
+const GROUP_PAYLOAD_CACHE_VERSION = 'v5';
// The default last-100 group bundle, keyed by group slug. The slug is the cache
// key (an `unstable_cache` argument), so one wrapper covers every group. A
diff --git a/web/lib/groups.test.ts b/web/lib/groups.test.ts
index 8c958de..f561481 100644
--- a/web/lib/groups.test.ts
+++ b/web/lib/groups.test.ts
@@ -74,7 +74,7 @@ describe.skipIf(!dockerAvailable())(
]);
});
- it('computes the random-access summary (ratio to fastest)', async () => {
+ it('computes the random-access summary (geomean ratio to fastest)', async () => {
const groups = await collectGroups();
const summary = expectDefined(
groups.find((g) => g.name === 'Random Access')?.summary,
@@ -86,8 +86,33 @@ describe.skipIf(!dockerAvailable())(
expect(summary.title).toBe('Random Access Performance');
expect(summary.rankings[0].name).toBe('vortex-file-compressed');
expect(summary.rankings[1].name).toBe('parquet');
- expect(summary.rankings[0].ratio).toBeCloseTo(1.0, 6);
- expect(summary.rankings[1].ratio).toBeCloseTo(2.0, 6);
+ // The fixture's newest commit has vortex=100_500 and parquet=201_000 on
+ // the one `taxi` chart, so the scores are the damped
+ // `(10 + value) / (10 + best)` ratios and both series cover every chart.
+ expect(summary.rankings[0].score).toBeCloseTo(1.0, 6);
+ expect(summary.rankings[1].score).toBeCloseTo(1.9999005, 6);
+ expect(summary.rankings[0].totalRuntime).toBeCloseTo(100_500, 6);
+ expect(summary.rankings[1].totalRuntime).toBeCloseTo(201_000, 6);
+ expect(summary.rankings.map((r) => [r.measured, r.total])).toEqual([
+ [1, 1],
+ [1, 1],
+ ]);
+ });
+
+ it('summarizes the vector-search group instead of leaving it blank', async () => {
+ const groups = await collectGroups();
+ const summary = expectDefined(
+ groups.find((g) => g.name === 'cohere-large-10m / partitioned')?.summary,
+ 'vector search summary',
+ );
+ if (summary.type !== 'vectorSearch') {
+ throw new Error(`expected vectorSearch summary, got ${summary.type}`);
+ }
+ expect(summary.title).toBe('Vector Search Performance');
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex-turboquant']);
+ expect(summary.rankings[0].score).toBeCloseTo(1.0, 6);
+ // The newest commit's value for the group's single threshold.
+ expect(summary.rankings[0].totalRuntime).toBeCloseTo(107_000, 6);
});
it('computes compression rankings with Parquet and Lance baselines', async () => {
@@ -170,8 +195,9 @@ describe.skipIf(!dockerAvailable())(
groups.find((g) => g.name === 'cohere-large-10m / partitioned'),
'vector-search group',
);
- // Vector-search groups carry neither a summary nor a description.
- expect(vector.summary).toBeUndefined();
+ // Vector-search groups now carry a summary (every group kind does), but
+ // still have no editorial description.
+ expect(vector.summary?.type).toBe('vectorSearch');
expect(vector.description).toBeUndefined();
});
@@ -343,7 +369,8 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
beforeEach(async () => {
await getPool().query(
- 'TRUNCATE compression_times, compression_sizes, query_measurements, commits',
+ `TRUNCATE compression_times, compression_sizes, query_measurements,
+ random_access_times, vector_search_runs, commits`,
);
});
@@ -366,6 +393,156 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
);
}
+ // One random-access measurement for a `{dataset}/{pattern}` chart. The
+ // producer (`benchmarks/random-access-bench`) writes `dataset` in exactly
+ // this shape, plus a legacy bare `taxi`.
+ async function insertRandomAccess(
+ sha: string,
+ dataset: string,
+ format: string,
+ valueNs: number,
+ ): Promise {
+ await getPool().query(
+ `INSERT INTO random_access_times
+ (measurement_id, commit_sha, dataset, format, value_ns, all_runtimes_ns)
+ VALUES ($1, $2, $3, $4, $5, '{1}'::bigint[])`,
+ [nextId(), sha, dataset, format, valueNs],
+ );
+ }
+
+ it('ranks random access over every chart, not the alphabetically first one', async () => {
+ // The bug this pins: the old summary walked the group's chart links and
+ // published the first populated chart's raw times under the group-wide
+ // title, so `lance` winning `feature-vectors/correlated` was reported as
+ // `lance` leading random access outright -- even though it is 3x slower on
+ // the two other charts. Ranking over all three charts reverses that.
+ const sha = 'a'.repeat(40);
+ await insertCommit(sha, '2026-04-23T12:00:00Z');
+ await insertRandomAccess(sha, 'feature-vectors/correlated', 'lance', 350_000);
+ await insertRandomAccess(
+ sha,
+ 'feature-vectors/correlated',
+ 'vortex-file-compressed',
+ 1_100_000,
+ );
+ await insertRandomAccess(sha, 'nested-structs/uniform', 'lance', 3_000_000);
+ await insertRandomAccess(sha, 'nested-structs/uniform', 'vortex-file-compressed', 1_000_000);
+ await insertRandomAccess(sha, 'taxi', 'lance', 3_000_000);
+ await insertRandomAccess(sha, 'taxi', 'vortex-file-compressed', 1_000_000);
+
+ const summary = expectDefined(
+ await collectGroupSummary({ k: 'RandomAccessGroup' }),
+ 'random-access summary',
+ );
+ if (summary.type !== 'randomAccess') {
+ throw new Error(`expected randomAccess summary, got ${summary.type}`);
+ }
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex-file-compressed', 'lance']);
+ // vortex: cbrt(1100000/350000 * 1 * 1); lance: cbrt(1 * 3 * 3).
+ expect(summary.rankings[0].score).toBeCloseTo(Math.cbrt(1_100_010 / 350_010), 5);
+ expect(summary.rankings[1].score).toBeCloseTo(Math.cbrt((3_000_010 / 1_000_010) ** 2), 5);
+ expect(summary.rankings[0].totalRuntime).toBeCloseTo(3_100_000, 6);
+ });
+
+ it('keeps an intermittently benchmarked format at its own latest run', async () => {
+ // `lance` runs less often than Vortex. Pinning every format to one global
+ // latest commit dropped it from the card entirely on any commit it skipped;
+ // each format is instead read at its own newest run per chart, the same
+ // freshness policy the compression summaries use.
+ const older = 'b'.repeat(40);
+ const newer = 'c'.repeat(40);
+ await insertCommit(older, '2026-04-22T12:00:00Z');
+ await insertCommit(newer, '2026-04-23T12:00:00Z');
+ await insertRandomAccess(older, 'taxi', 'lance', 4_000_000);
+ await insertRandomAccess(older, 'taxi', 'vortex-file-compressed', 2_000_000);
+ // The newer commit has no `lance` row.
+ await insertRandomAccess(newer, 'taxi', 'vortex-file-compressed', 1_000_000);
+
+ const summary = expectDefined(
+ await collectGroupSummary({ k: 'RandomAccessGroup' }),
+ 'random-access summary',
+ );
+ if (summary.type !== 'randomAccess') {
+ throw new Error(`expected randomAccess summary, got ${summary.type}`);
+ }
+ const byName = new Map(summary.rankings.map((r) => [r.name, r]));
+ // Vortex is read at the newer commit, lance at its own older one.
+ expect(byName.get('vortex-file-compressed')?.totalRuntime).toBeCloseTo(1_000_000, 6);
+ expect(byName.get('lance')?.totalRuntime).toBeCloseTo(4_000_000, 6);
+ });
+
+ it('penalizes a format that skipped a chart instead of scoring it on a subset', async () => {
+ // A format measured only where it wins would otherwise take #1 on a
+ // one-chart geomean. `lance` is fastest on the chart it ran and absent from
+ // the other; the missing chart is imputed `max(itsWorstChart, 0) * 2`, which
+ // is what drops it behind the format measured everywhere.
+ const sha = 'd'.repeat(40);
+ await insertCommit(sha, '2026-04-23T12:00:00Z');
+ await insertRandomAccess(sha, 'feature-vectors/correlated', 'lance', 100_000);
+ await insertRandomAccess(sha, 'feature-vectors/correlated', 'vortex-file-compressed', 200_000);
+ await insertRandomAccess(sha, 'taxi', 'vortex-file-compressed', 50_000);
+
+ const summary = expectDefined(
+ await collectGroupSummary({ k: 'RandomAccessGroup' }),
+ 'random-access summary',
+ );
+ if (summary.type !== 'randomAccess') {
+ throw new Error(`expected randomAccess summary, got ${summary.type}`);
+ }
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex-file-compressed', 'lance']);
+ const byName = new Map(summary.rankings.map((r) => [r.name, r]));
+ // lance: sqrt(1 * (10 + 200_000) / (10 + 50_000)) with penalty
+ // max(100_000, 0) * 2 = 200_000 on the chart it skipped.
+ expect(byName.get('lance')?.score).toBeCloseTo(Math.sqrt(200_010 / 50_010), 6);
+ expect(byName.get('lance')?.measured).toBe(1);
+ expect(byName.get('lance')?.total).toBe(2);
+ // `totalRuntime` sums only the charts it actually ran; the penalty is a
+ // scoring device, not a reported measurement.
+ expect(byName.get('lance')?.totalRuntime).toBeCloseTo(100_000, 6);
+ expect(byName.get('vortex-file-compressed')?.score).toBeCloseTo(
+ Math.sqrt(200_010 / 100_010),
+ 6,
+ );
+ expect(byName.get('vortex-file-compressed')?.measured).toBe(2);
+ });
+
+ it('summarizes a vector-search group across its thresholds', async () => {
+ const sha = 'e'.repeat(40);
+ await insertCommit(sha, '2026-04-23T12:00:00Z');
+ const rows: ReadonlyArray = [
+ ['vortex-turboquant', 0.5, 1_000],
+ ['vortex-turboquant', 0.75, 2_000],
+ ['vortex-flat', 0.5, 2_000],
+ ['vortex-flat', 0.75, 8_000],
+ ];
+ for (const [flavor, threshold, valueNs] of rows) {
+ await getPool().query(
+ `INSERT INTO vector_search_runs
+ (measurement_id, commit_sha, dataset, layout, flavor, threshold, value_ns,
+ all_runtimes_ns, matches, rows_scanned, bytes_scanned, iterations)
+ VALUES ($1, $2, 'cohere-large-10m', 'partitioned', $3, $4, $5,
+ '{1}'::bigint[], 42, 1000000, 5000000, 1)`,
+ [nextId(), sha, flavor, threshold, valueNs],
+ );
+ }
+
+ const summary = expectDefined(
+ await collectGroupSummary({
+ k: 'VectorSearchGroup',
+ dataset: 'cohere-large-10m',
+ layout: 'partitioned',
+ }),
+ 'vector-search summary',
+ );
+ if (summary.type !== 'vectorSearch') {
+ throw new Error(`expected vectorSearch summary, got ${summary.type}`);
+ }
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex-turboquant', 'vortex-flat']);
+ expect(summary.rankings[0].score).toBeCloseTo(1.0, 6);
+ // sqrt((2010/1010) * (8010/2010)).
+ expect(summary.rankings[1].score).toBeCloseTo(Math.sqrt((2010 / 1010) * (8010 / 2010)), 6);
+ });
+
it('keeps a sub-second latest commit timestamp (no whole-second truncation)', async () => {
// A single commit whose timestamp carries microseconds. The pre-fix code
// rendered MAX(ts) to whole-second text and rebound it with exact
@@ -377,7 +554,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
await insertCompSizePair(sha, 1_000, 4_000);
const time = expectDefined(
- await collectGroupSummary({ k: 'CompressionTimeGroup' }, []),
+ await collectGroupSummary({ k: 'CompressionTimeGroup' }),
'compression-time summary',
);
if (time.type !== 'compression') {
@@ -392,7 +569,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
expect(timeByKey.get('decode:parquet')?.ratio).toBeCloseTo(1.0, 6);
const size = expectDefined(
- await collectGroupSummary({ k: 'CompressionSizeGroup' }, []),
+ await collectGroupSummary({ k: 'CompressionSizeGroup' }),
'compression-size summary',
);
if (size.type !== 'compressionSize') {
@@ -418,7 +595,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
await insertCompSizePair(higherSha, 2_000, 8_000);
const time = expectDefined(
- await collectGroupSummary({ k: 'CompressionTimeGroup' }, []),
+ await collectGroupSummary({ k: 'CompressionTimeGroup' }),
'compression-time summary',
);
if (time.type !== 'compression') {
@@ -433,7 +610,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
expect(vortexTime.ratio).toBeCloseTo(4.0, 6);
const size = expectDefined(
- await collectGroupSummary({ k: 'CompressionSizeGroup' }, []),
+ await collectGroupSummary({ k: 'CompressionSizeGroup' }),
'compression-size summary',
);
if (size.type !== 'compressionSize') {
@@ -459,7 +636,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
await insertCompTimePair(newer, 'decode', 1_000, 2_000);
const summary = expectDefined(
- await collectGroupSummary({ k: 'CompressionTimeGroup' }, []),
+ await collectGroupSummary({ k: 'CompressionTimeGroup' }),
'compression-time summary',
);
if (summary.type !== 'compression') {
@@ -478,7 +655,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
await insertCompTimePair(sha, 'decode', 1_000, 2_000);
const summary = expectDefined(
- await collectGroupSummary({ k: 'CompressionTimeGroup' }, []),
+ await collectGroupSummary({ k: 'CompressionTimeGroup' }),
'compression-time summary',
);
if (summary.type !== 'compression') {
@@ -504,16 +681,13 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg
await insertQuery(sha, 1, 'duckdb', 'parquet', 50_000);
const summary = expectDefined(
- await collectGroupSummary(
- {
- k: 'QueryGroup',
- dataset: 'tpch',
- dataset_variant: null,
- scale_factor: '1',
- storage: 'nvme',
- },
- [],
- ),
+ await collectGroupSummary({
+ k: 'QueryGroup',
+ dataset: 'tpch',
+ dataset_variant: null,
+ scale_factor: '1',
+ storage: 'nvme',
+ }),
'query summary',
);
if (summary.type !== 'queryBenchmark') {
@@ -627,16 +801,13 @@ describe.skipIf(!dockerAvailable())(
});
it('prefers a stamped row over a newer NULL-stamped row and keeps all-NULL series', async () => {
- const summary = await collectGroupSummary(
- {
- k: 'QueryGroup',
- dataset: 'tpch',
- dataset_variant: null,
- scale_factor: '1',
- storage: 'nvme',
- },
- [],
- );
+ const summary = await collectGroupSummary({
+ k: 'QueryGroup',
+ dataset: 'tpch',
+ dataset_variant: null,
+ scale_factor: '1',
+ storage: 'nvme',
+ });
if (summary === null || summary.type !== 'queryBenchmark') {
throw new Error(`expected queryBenchmark summary, got ${summary?.type}`);
}
@@ -653,16 +824,13 @@ describe.skipIf(!dockerAvailable())(
});
it('enumerates the format, engine, and query_idx successor branches', async () => {
- const summary = await collectGroupSummary(
- {
- k: 'QueryGroup',
- dataset: 'tpch',
- dataset_variant: null,
- scale_factor: '1',
- storage: 'nvme',
- },
- [],
- );
+ const summary = await collectGroupSummary({
+ k: 'QueryGroup',
+ dataset: 'tpch',
+ dataset_variant: null,
+ scale_factor: '1',
+ storage: 'nvme',
+ });
if (summary === null || summary.type !== 'queryBenchmark') {
throw new Error(`expected queryBenchmark summary, got ${summary?.type}`);
}
diff --git a/web/lib/queries.ts b/web/lib/queries.ts
index a84035b..f789196 100644
--- a/web/lib/queries.ts
+++ b/web/lib/queries.ts
@@ -642,10 +642,11 @@ export interface ChartLink {
/**
* One group: a display name, a permalink slug, the chart links inside it, and
- * an optional v2-compatible rollup [`Summary`] plus editorial description.
- * `summary` and `description` are left `undefined` (so `JSON.stringify` drops
- * them) when absent, the TS analogue of serde `skip_serializing_if =
- * "Option::is_none"`.
+ * a rollup [`Summary`] plus an optional editorial description. Every group
+ * kind has a summary (see `summary.ts`), so `summary` is absent only for a
+ * group whose fact rows are not yet usable. `summary` and `description` are
+ * left `undefined` (so `JSON.stringify` drops them) when absent, the TS
+ * analogue of serde `skip_serializing_if = "Option::is_none"`.
*/
export interface Group {
name: string;
@@ -1135,7 +1136,7 @@ export async function collectGroups(): Promise {
// ~64 groups (PR-5.1.5 fix e).
await mapWithConcurrency(groups, SUMMARY_CONCURRENCY, async (group) => {
const key = groupKeyFromSlug(group.slug);
- const summary = await collectGroupSummary(key, group.charts);
+ const summary = await collectGroupSummary(key);
if (summary !== null) {
group.summary = summary;
}
diff --git a/web/lib/summary.test.ts b/web/lib/summary.test.ts
index 6310188..e1b95d9 100644
--- a/web/lib/summary.test.ts
+++ b/web/lib/summary.test.ts
@@ -9,8 +9,13 @@ vi.mock('./db', () => ({
getPool: () => ({ query }),
}));
+import { FAMILIES } from './families';
+import type { GroupKey } from './slug';
import { collectGroupSummary } from './summary';
+/** Every group discriminant, from the fact-table registry. */
+const GROUP_KINDS = FAMILIES.map((family) => family.groupKind);
+
describe('compression summaries', () => {
beforeEach(() => {
query.mockReset();
@@ -26,7 +31,7 @@ describe('compression summaries', () => {
],
});
- const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }, []);
+ const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' });
if (summary === null || summary.type !== 'compressionSize') {
throw new Error('expected a compressionSize summary');
}
@@ -40,8 +45,8 @@ describe('compression summaries', () => {
it('applies one extensible snapshot policy to timings and sizes', async () => {
query.mockResolvedValue({ rows: [] });
- await collectGroupSummary({ k: 'CompressionTimeGroup' }, []);
- await collectGroupSummary({ k: 'CompressionSizeGroup' }, []);
+ await collectGroupSummary({ k: 'CompressionTimeGroup' });
+ await collectGroupSummary({ k: 'CompressionSizeGroup' });
expect(query).toHaveBeenCalledTimes(2);
const expectedParams = [
@@ -67,3 +72,167 @@ describe('compression summaries', () => {
}
});
});
+
+describe('timing summaries (shared ranking model)', () => {
+ beforeEach(() => {
+ query.mockReset();
+ });
+
+ it('ranks random access across every chart, not the first one', async () => {
+ // The regression: the old summary published one chart's raw times under the
+ // group-wide title. `lance` wins `feature-vectors/correlated` outright and
+ // loses the other two charts 3x; the group ranking must reflect all three.
+ query.mockResolvedValueOnce({
+ rows: [
+ { bucket: 'feature-vectors/correlated', series: 'lance', value: 350_000 },
+ { bucket: 'feature-vectors/correlated', series: 'vortex', value: 1_100_000 },
+ { bucket: 'nested-structs/uniform', series: 'lance', value: 3_000_000 },
+ { bucket: 'nested-structs/uniform', series: 'vortex', value: 1_000_000 },
+ { bucket: 'taxi', series: 'lance', value: 3_000_000 },
+ { bucket: 'taxi', series: 'vortex', value: 1_000_000 },
+ ],
+ });
+
+ const summary = await collectGroupSummary({ k: 'RandomAccessGroup' });
+ if (summary === null || summary.type !== 'randomAccess') {
+ throw new Error('expected a randomAccess summary');
+ }
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex', 'lance']);
+ expect(summary.rankings[0].score).toBeCloseTo(Math.cbrt(1_100_010 / 350_010), 6);
+ expect(summary.rankings[1].score).toBeCloseTo(Math.cbrt((3_000_010 / 1_000_010) ** 2), 6);
+ expect(summary.rankings[0].totalRuntime).toBeCloseTo(3_100_000, 6);
+ expect(summary.rankings.map((r) => r.measured)).toEqual([3, 3]);
+ expect(summary.rankings.map((r) => r.total)).toEqual([3, 3]);
+ });
+
+ it('reads each random-access format at its own newest run', async () => {
+ query.mockResolvedValue({ rows: [] });
+ await collectGroupSummary({ k: 'RandomAccessGroup' });
+ const [text] = query.mock.calls[0] as [string, unknown[] | undefined];
+ // Per-series freshness, not one global latest commit: a format that skipped
+ // the newest commit stays on the card at its own last run.
+ expect(text).toContain('DISTINCT ON (r.dataset, r.format)');
+ expect(text).toContain('ORDER BY r.dataset, r.format, c.timestamp DESC');
+ expect(text).not.toContain('MAX(c2.timestamp)');
+ });
+
+ it('penalizes a series that skipped a bucket and reports its coverage', async () => {
+ query.mockResolvedValueOnce({
+ rows: [
+ { bucket: 'feature-vectors/correlated', series: 'lance', value: 100_000 },
+ { bucket: 'feature-vectors/correlated', series: 'vortex', value: 200_000 },
+ { bucket: 'taxi', series: 'vortex', value: 50_000 },
+ ],
+ });
+
+ const summary = await collectGroupSummary({ k: 'RandomAccessGroup' });
+ if (summary === null || summary.type !== 'randomAccess') {
+ throw new Error('expected a randomAccess summary');
+ }
+ const byName = new Map(summary.rankings.map((r) => [r.name, r]));
+ // lance skipped `taxi`, so that bucket scores max(100_000, 0) * 2 against
+ // taxi's best of 50_000 -- which is what keeps it behind `vortex`.
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex', 'lance']);
+ expect(byName.get('lance')?.score).toBeCloseTo(Math.sqrt(200_010 / 50_010), 6);
+ expect(byName.get('lance')?.measured).toBe(1);
+ expect(byName.get('lance')?.total).toBe(2);
+ // `totalRuntime` reports measured time only; the penalty never enters it.
+ expect(byName.get('lance')?.totalRuntime).toBeCloseTo(100_000, 6);
+ expect(byName.get('vortex')?.measured).toBe(2);
+ });
+
+ it('does not reward a series for skipping a slow bucket', async () => {
+ query.mockResolvedValueOnce({
+ rows: [
+ { bucket: 'easy', series: 'partial', value: 100_000 },
+ { bucket: 'easy', series: 'complete', value: 110_000 },
+ { bucket: 'slow', series: 'complete', value: 100_000_000 },
+ ],
+ });
+
+ const summary = await collectGroupSummary({ k: 'RandomAccessGroup' });
+ if (summary === null || summary.type !== 'randomAccess') {
+ throw new Error('expected a randomAccess summary');
+ }
+ const byName = new Map(summary.rankings.map((r) => [r.name, r]));
+ // The absolute penalty for `partial` is 200_000ns. That value is faster
+ // than the measured 100_000_000ns best on `slow`, so the 2x ratio floor
+ // must prevent the absent bucket from improving the partial series' score.
+ expect(summary.rankings.map((r) => r.name)).toEqual(['complete', 'partial']);
+ expect(byName.get('partial')?.score).toBeCloseTo(Math.sqrt(2), 6);
+ expect(byName.get('partial')?.measured).toBe(1);
+ expect(byName.get('partial')?.total).toBe(2);
+ });
+
+ it('summarizes a vector-search group across its thresholds', async () => {
+ query.mockResolvedValueOnce({
+ rows: [
+ { bucket: 0.5, series: 'vortex-turboquant', value: 1_000 },
+ { bucket: 0.75, series: 'vortex-turboquant', value: 2_000 },
+ { bucket: 0.5, series: 'vortex-flat', value: 2_000 },
+ { bucket: 0.75, series: 'vortex-flat', value: 8_000 },
+ ],
+ });
+
+ const summary = await collectGroupSummary({
+ k: 'VectorSearchGroup',
+ dataset: 'cohere-large-10m',
+ layout: 'partitioned',
+ });
+ if (summary === null || summary.type !== 'vectorSearch') {
+ throw new Error('expected a vectorSearch summary');
+ }
+ expect(summary.title).toBe('Vector Search Performance');
+ expect(summary.rankings.map((r) => r.name)).toEqual(['vortex-turboquant', 'vortex-flat']);
+ expect(summary.rankings[0].score).toBeCloseTo(1.0, 6);
+ expect(summary.rankings[1].score).toBeCloseTo(Math.sqrt((2010 / 1010) * (8010 / 2010)), 6);
+ expect(summary.rankings[1].totalRuntime).toBeCloseTo(10_000, 6);
+ });
+
+ it('summarizes every query group, with no dataset allowlist', async () => {
+ // `spatialbench` (and every other suite outside the retired v2 five) used to
+ // fall through to `null` and render no card at all.
+ for (const dataset of ['spatialbench', 'fineweb', 'gharchive', 'appian', 'tpch']) {
+ query.mockResolvedValueOnce({
+ rows: [
+ { query_idx: 1, series: 'datafusion:vortex', value_ns: 1_000 },
+ { query_idx: 1, series: 'duckdb:parquet', value_ns: 2_000 },
+ ],
+ });
+ const summary = await collectGroupSummary({
+ k: 'QueryGroup',
+ dataset,
+ dataset_variant: null,
+ scale_factor: null,
+ storage: 'nvme',
+ });
+ expect(summary?.type).toBe('queryBenchmark');
+ }
+ });
+
+ it('returns a summary for every group kind', async () => {
+ // The default-on contract: a new suite landing in any fact table gets a
+ // card without a summary-side change. Only an empty result yields `null`.
+ const keys: GroupKey[] = [
+ {
+ k: 'QueryGroup',
+ dataset: 'newsuite',
+ dataset_variant: null,
+ scale_factor: null,
+ storage: 'nvme',
+ },
+ { k: 'CompressionTimeGroup' },
+ { k: 'CompressionSizeGroup' },
+ { k: 'RandomAccessGroup' },
+ { k: 'VectorSearchGroup', dataset: 'd', layout: 'l' },
+ ];
+ expect(keys.map((key) => key.k).sort()).toEqual([...GROUP_KINDS].sort());
+ for (const key of keys) {
+ query.mockResolvedValue({ rows: [] });
+ // Every kind reaches SQL; none short-circuits to `null` on its key alone.
+ query.mockClear();
+ await collectGroupSummary(key);
+ expect(query).toHaveBeenCalled();
+ }
+ });
+});
diff --git a/web/lib/summary.ts b/web/lib/summary.ts
index 30579bc..029bb3f 100644
--- a/web/lib/summary.ts
+++ b/web/lib/summary.ts
@@ -5,8 +5,16 @@
* Per-group summary rollups.
*
* Each `collect*Summary` runs focused SQL queries and returns one [`Summary`]
- * variant. Query summaries use a v2 dataset allowlist. Compression summaries
- * compare the configured formats with Parquet.
+ * variant. Compression summaries compare the configured formats with Parquet.
+ *
+ * **Every group kind has a summary.** There is no allowlist gate: a benchmark
+ * suite that lands in one of the five fact tables gets a rollup card the moment
+ * its first rows are ingested. The timing-shaped families (query, random
+ * access, vector search) all rank through the one [`rankSeries`] model below,
+ * so a new suite in any of them needs no summary code at all. Adding a sixth
+ * fact table is the only case that needs a new arm here, and
+ * [`collectGroupSummary`]'s exhaustive switch makes omitting it a compile
+ * error rather than a silently missing card.
*
* Behaviour-preservation notes (substrate migration, DuckDB -> Postgres):
* - Nullable-dim equality (`dataset_variant` / `scale_factor`) in the
@@ -55,26 +63,33 @@ function compressionSummaryQueryParams(): [string[], string[], string, string] {
];
}
-/** One random-access summary row. */
-export interface RandomAccessRanking {
- /** Series name, normally the physical format. */
- name: string;
- /** Latest measured time in nanoseconds. */
- time: number;
- /** Ratio to the fastest series in the same chart. */
- ratio: number;
-}
-
-/** One query-benchmark summary row. */
-export interface QueryRanking {
- /** Series name, normally `engine:format`. */
+/**
+ * One row of a timing-benchmark ranking (query, random access, vector search).
+ *
+ * `score` is the headline number: the geomean of this series' time ratio to
+ * the fastest series, taken over every bucket in the group (queries for a
+ * query suite, `dataset/pattern` charts for random access, thresholds for
+ * vector search). A single bucket's absolute time is deliberately NOT the
+ * headline -- a summary that quoted one arbitrary chart's numbers reads as a
+ * claim about the whole group and is wrong whenever the group's charts
+ * disagree.
+ */
+export interface SeriesRanking {
+ /** Series name: `engine:format` for queries, the format or flavor otherwise. */
name: string;
- /** Geomean ratio to the fastest observed value per query. */
+ /** Geomean ratio to the fastest series per bucket. Lower is better. */
score: number;
- /** Sum of latest runtimes for the queries this series has. */
+ /** Sum of the latest times over the buckets this series was measured in. */
totalRuntime: number;
+ /** Buckets this series has a measurement for. */
+ measured: number;
+ /** Buckets in the group, i.e. the denominator `measured` is out of. */
+ total: number;
}
+/** @deprecated Prefer [`SeriesRanking`]; kept as the historical spelling. */
+export type QueryRanking = SeriesRanking;
+
/** One format and operation in the compression throughput summary. */
export interface CompressionRanking {
/** On-disk format. */
@@ -101,7 +116,7 @@ export type Summary =
| {
type: 'randomAccess';
title: string;
- rankings: RandomAccessRanking[];
+ rankings: SeriesRanking[];
explanation: string;
}
| {
@@ -119,48 +134,33 @@ export type Summary =
| {
type: 'queryBenchmark';
title: string;
- rankings: QueryRanking[];
+ rankings: SeriesRanking[];
+ explanation: string;
+ }
+ | {
+ type: 'vectorSearch';
+ title: string;
+ rankings: SeriesRanking[];
explanation: string;
};
/**
- * Compute the summary for one group, if its kind has one.
- * `charts` is only consulted for the random-access path (which scans its chart
- * links for the latest populated dataset); the other paths query their fact
- * table directly. The structural `{ name }[]` accepts a `ChartLink[]`.
+ * Compute the summary for one group. Every group kind has one, so this returns
+ * `null` only when the group has no usable rows yet (a brand-new suite between
+ * its first ingest and its first complete measurement).
*/
-export function collectGroupSummary(
- key: GroupKey,
- charts: readonly { readonly name: string }[],
-): Promise {
+export function collectGroupSummary(key: GroupKey): Promise {
switch (key.k) {
case 'QueryGroup':
- if (queryGroupHasV2Summary(key.dataset)) {
- return collectQuerySummary(key.dataset, key.dataset_variant, key.scale_factor, key.storage);
- }
- return Promise.resolve(null);
+ return collectQuerySummary(key.dataset, key.dataset_variant, key.scale_factor, key.storage);
case 'CompressionTimeGroup':
return collectCompressionSummary();
case 'CompressionSizeGroup':
return collectCompressionSizeSummary();
case 'RandomAccessGroup':
- return collectRandomAccessSummary(charts);
+ return collectRandomAccessSummary();
case 'VectorSearchGroup':
- return Promise.resolve(null);
- }
-}
-
-/** The v2 dataset allowlist for which a query group carries a summary. */
-function queryGroupHasV2Summary(dataset: string): boolean {
- switch (dataset) {
- case 'clickbench':
- case 'statpopgen':
- case 'polarsignals':
- case 'tpch':
- case 'tpcds':
- return true;
- default:
- return false;
+ return collectVectorSearchSummary(key.dataset, key.layout);
}
}
@@ -181,62 +181,225 @@ function geoMean(values: readonly number[]): number | null {
return n > 0 ? Math.exp(sumLn / n) : null;
}
-async function collectRandomAccessSummary(
- charts: readonly { readonly name: string }[],
-): Promise {
- // Scan the group's chart links in order; the first chart with valid rows at
- // its latest commit wins (matching the Rust early-return loop).
- const text = `
- SELECT r.format AS name, r.value_ns::float8 AS value
- FROM random_access_times r
- JOIN commits c USING (commit_sha)
- WHERE r.dataset = $1
- AND r.value_ns > 0
- AND c.timestamp = (
- SELECT MAX(c2.timestamp)
- FROM random_access_times r2
- JOIN commits c2 USING (commit_sha)
- WHERE r2.dataset = $2
- AND r2.value_ns > 0
- )
- ORDER BY r.value_ns, r.format
- `;
- for (const chart of charts) {
- const rows = (
- await getPool().query<{ name: string; value: number }>(text, [chart.name, chart.name])
- ).rows;
- const rankings: RandomAccessRanking[] = rows.map((row) => ({
- name: row.name,
- time: row.value,
- ratio: 0,
- }));
- if (rankings.length === 0) {
+/**
+ * The v2 penalty floor for query suites, in nanoseconds. A series missing a
+ * query is imputed `max(itsWorstQuery, 300us) * 2`; the floor stops a suite of
+ * uniformly fast queries from making a missing result nearly free.
+ */
+const QUERY_PENALTY_FLOOR_NS = 300_000;
+
+/** One measured point feeding [`rankSeries`]. */
+interface SeriesSample {
+ /** Series being ranked (`engine:format`, a format, or a flavor). */
+ series: string;
+ /** The thing being measured across series: a query, a chart, a threshold. */
+ bucket: K;
+ /** Latest value for `(series, bucket)`, in nanoseconds. */
+ value: number;
+}
+
+/**
+ * Rank timing series by the geomean of their ratio to the fastest series in
+ * each bucket, with v2's missing-series penalty.
+ *
+ * This is the one ranking model behind the query, random-access, and
+ * vector-search summaries. Two properties matter for a summary card:
+ *
+ * - **Every bucket counts.** Ranking on one bucket's absolute times (which is
+ * what the random-access summary used to do -- it took whichever chart
+ * sorted first and quoted its raw numbers) states a group-wide conclusion
+ * from a single chart. `lance` leading `feature-vectors/correlated` says
+ * nothing about `nested-structs/uniform`.
+ * - **A missing bucket is not a free win.** A series measured on only the
+ * buckets it happens to win would otherwise outrank one measured
+ * everywhere. Where a series has no value the bucket contributes
+ * `max(itsWorstBucket, penaltyFloorNs) * 2` instead. `measured`/`total` on
+ * each row reports how much of the score was imputed.
+ *
+ * The `(10 + value) / (10 + best)` ratio (rather than `value / best`) is v2's,
+ * damping sub-10ns noise; it is preserved because the shipped query scores are
+ * pinned to it. Random-access and vector-search summaries also set a 2x floor
+ * for a missing bucket. The floor prevents a penalty derived from a fast bucket
+ * from beating a real measurement on a slower bucket. Query summaries keep a
+ * zero floor to preserve the shipped v2 scores.
+ */
+function rankSeries(
+ samples: readonly SeriesSample[],
+ compareBuckets: (a: K, b: K) => number,
+ penaltyFloorNs: number,
+ missingRatioFloor: number,
+): SeriesRanking[] {
+ const buckets = new Map();
+ const valuesBySeries = new Map>();
+ for (const sample of samples) {
+ if (!(sample.value > 0) || !Number.isFinite(sample.value)) {
continue;
}
- // Streaming min (loop, not a `Math.min(...)` spread) for consistency with
- // `collectCompressionSizeSummary` and to avoid a large-array call-argument
- // cliff; the Rust source uses `reduce(f64::min)` here.
- let minTime = Infinity;
- for (const r of rankings) {
- minTime = Math.min(minTime, r.time);
+ const bucketKey = String(sample.bucket);
+ buckets.set(bucketKey, sample.bucket);
+ let series = valuesBySeries.get(sample.series);
+ if (series === undefined) {
+ series = new Map();
+ valuesBySeries.set(sample.series, series);
}
- if (minTime <= 0 || !Number.isFinite(minTime)) {
+ series.set(bucketKey, sample.value);
+ }
+ if (valuesBySeries.size === 0) {
+ return [];
+ }
+
+ // Sorted buckets match the Rust `BTreeSet` iteration order for query suites.
+ const sortedBuckets = [...buckets.entries()].sort((a, b) => compareBuckets(a[1], b[1]));
+ const bestByBucket = new Map();
+ for (const [bucketKey] of sortedBuckets) {
+ let best = Infinity;
+ for (const series of valuesBySeries.values()) {
+ const value = series.get(bucketKey);
+ if (value !== undefined && value < best) {
+ best = value;
+ }
+ }
+ if (Number.isFinite(best)) {
+ bestByBucket.set(bucketKey, best);
+ }
+ }
+
+ const rankings: SeriesRanking[] = [];
+ // Sorted series keys match the Rust `BTreeMap` iteration order.
+ for (const name of [...valuesBySeries.keys()].sort(compareCodeUnits)) {
+ const bucketValues = valuesBySeries.get(name);
+ if (bucketValues === undefined) {
continue;
}
- for (const r of rankings) {
- r.ratio = r.time / minTime;
+ let totalRuntime = 0;
+ let maxRuntime = -Infinity;
+ for (const [bucketKey] of sortedBuckets) {
+ const value = bucketValues.get(bucketKey);
+ if (value === undefined) {
+ continue;
+ }
+ totalRuntime += value;
+ if (value > maxRuntime) {
+ maxRuntime = value;
+ }
}
- rankings.sort((a, b) =>
- a.time < b.time ? -1 : a.time > b.time ? 1 : compareCodeUnits(a.name, b.name),
- );
- return {
- type: 'randomAccess',
- title: 'Random Access Performance',
- rankings,
- explanation: 'Random access time | Ratio to fastest (lower is better)',
- };
+ if (!Number.isFinite(maxRuntime)) {
+ continue;
+ }
+ const penalty = Math.max(maxRuntime, penaltyFloorNs) * 2;
+ const ratios: number[] = [];
+ for (const [bucketKey] of sortedBuckets) {
+ const base = bestByBucket.get(bucketKey);
+ if (base === undefined) {
+ continue;
+ }
+ const measuredValue = bucketValues.get(bucketKey);
+ const ratio = (10 + (measuredValue ?? penalty)) / (10 + base);
+ ratios.push(measuredValue === undefined ? Math.max(ratio, missingRatioFloor) : ratio);
+ }
+ const score = geoMean(ratios);
+ if (score === null) {
+ continue;
+ }
+ rankings.push({
+ name,
+ score,
+ totalRuntime,
+ measured: bucketValues.size,
+ total: sortedBuckets.length,
+ });
+ }
+ rankings.sort((a, b) =>
+ a.score < b.score ? -1 : a.score > b.score ? 1 : compareCodeUnits(a.name, b.name),
+ );
+ return rankings;
+}
+
+/**
+ * The random-access rollup, over every `(dataset, pattern)` chart in the group.
+ *
+ * Two things this deliberately does NOT do, both of which the previous
+ * implementation did:
+ *
+ * - It does not summarize one chart. The old query walked the group's chart
+ * links and returned the first that had rows -- in practice always the
+ * alphabetically first `dataset/pattern` -- then published its raw times
+ * under the group-wide title "Random Access Performance". The producer
+ * (`benchmarks/random-access-bench`) emits `dataset` as `{dataset}/{pattern}`
+ * plus the legacy bare `taxi`, so that was one of nine charts speaking for
+ * all nine.
+ * - It does not pin every format to one global latest commit. `format` is
+ * ranked from its own newest run per chart, the same freshness policy the
+ * compression summaries apply to intermittently benchmarked formats such as
+ * `lance`: a format that skipped the newest commit is compared as of when it
+ * last ran instead of vanishing from the card.
+ *
+ * `random_access_times` holds one row per `(commit_sha, dataset, format)` and
+ * is the smallest fact table, so the per-series `DISTINCT ON` descent is cheap.
+ */
+async function collectRandomAccessSummary(): Promise {
+ const text = `
+ SELECT DISTINCT ON (r.dataset, r.format)
+ r.dataset AS bucket,
+ r.format AS series,
+ r.value_ns::float8 AS value
+ FROM random_access_times r
+ JOIN commits c USING (commit_sha)
+ WHERE r.value_ns > 0
+ ORDER BY r.dataset, r.format, c.timestamp DESC, r.commit_sha DESC
+ `;
+ const rows = (await getPool().query<{ bucket: string; series: string; value: number }>(text))
+ .rows;
+ const rankings = rankSeries(rows, compareCodeUnits, 0, 2);
+ if (rankings.length === 0) {
+ return null;
}
- return null;
+ return {
+ type: 'randomAccess',
+ title: 'Random Access Performance',
+ rankings,
+ explanation: 'Geomean of take time ratio to fastest across every chart (lower is better)',
+ };
+}
+
+/**
+ * The vector-search rollup for one `(dataset, layout)` group, ranking flavors
+ * across the group's thresholds. Vector-search groups previously carried no
+ * summary at all; they rank through the same model as every other timing
+ * family, with the threshold as the bucket.
+ */
+async function collectVectorSearchSummary(
+ dataset: string,
+ layout: string,
+): Promise {
+ const text = `
+ SELECT DISTINCT ON (v.threshold, v.flavor)
+ v.threshold::float8 AS bucket,
+ v.flavor AS series,
+ v.value_ns::float8 AS value
+ FROM vector_search_runs v
+ JOIN commits c USING (commit_sha)
+ WHERE v.dataset = $1
+ AND v.layout = $2
+ AND v.value_ns > 0
+ ORDER BY v.threshold, v.flavor, c.timestamp DESC, v.commit_sha DESC
+ `;
+ const rows = (
+ await getPool().query<{ bucket: number; series: string; value: number }>(text, [
+ dataset,
+ layout,
+ ])
+ ).rows;
+ const rankings = rankSeries(rows, (a, b) => a - b, 0, 2);
+ if (rankings.length === 0) {
+ return null;
+ }
+ return {
+ type: 'vectorSearch',
+ title: 'Vector Search Performance',
+ rankings,
+ explanation: 'Geomean of scan time ratio to fastest across thresholds (lower is better)',
+ };
}
async function collectCompressionSummary(): Promise {
@@ -470,9 +633,9 @@ async function collectQuerySummary(
scaleFactor: string | null,
storage: string,
): Promise {
- // Latest value per (query_idx, engine, format), then v2's missing-series
- // penalty model: each series scores the geomean of `(10 + value) / (10 +
- // best)` over every query, imputing a penalty where the series has no value.
+ // Latest value per (query_idx, engine, format), then the shared
+ // [`rankSeries`] model with the query bucket = `query_idx` and v2's
+ // `QUERY_PENALTY_FLOOR_NS` floor.
//
// "Latest per series" is a recursive-CTE skip scan (loose index scan) over the
// covering index `idx_query_measurements_summary` (dataset, dataset_variant,
@@ -636,77 +799,12 @@ async function collectQuerySummary(
await getPool().query<{ query_idx: number; series: string; value_ns: number }>(text, params)
).rows;
- const queries = new Set();
- const valuesBySeries = new Map>();
- for (const row of rows) {
- queries.add(row.query_idx);
- let series = valuesBySeries.get(row.series);
- if (series === undefined) {
- series = new Map();
- valuesBySeries.set(row.series, series);
- }
- series.set(row.query_idx, row.value_ns);
- }
- if (valuesBySeries.size === 0) {
- return null;
- }
-
- // Sorted query indices match the Rust `BTreeSet` iteration order.
- const sortedQueries = [...queries].sort((a, b) => a - b);
- const bestByQuery = new Map();
- for (const queryIdx of sortedQueries) {
- let best = Infinity;
- for (const series of valuesBySeries.values()) {
- const value = series.get(queryIdx);
- if (value !== undefined && value < best) {
- best = value;
- }
- }
- if (Number.isFinite(best)) {
- bestByQuery.set(queryIdx, best);
- }
- }
-
- const rankings: QueryRanking[] = [];
- // Sorted series keys match the Rust `BTreeMap` iteration order.
- for (const name of [...valuesBySeries.keys()].sort(compareCodeUnits)) {
- const queryValues = valuesBySeries.get(name);
- if (queryValues === undefined) {
- continue;
- }
- let totalRuntime = 0;
- for (const queryIdx of [...queryValues.keys()].sort((a, b) => a - b)) {
- totalRuntime += queryValues.get(queryIdx) ?? 0;
- }
- let maxRuntime = -Infinity;
- for (const value of queryValues.values()) {
- if (value > maxRuntime) {
- maxRuntime = value;
- }
- }
- if (!Number.isFinite(maxRuntime)) {
- continue;
- }
- const penalty = Math.max(maxRuntime, 300_000) * 2;
- const ratios: number[] = [];
- for (const queryIdx of sortedQueries) {
- const base = bestByQuery.get(queryIdx);
- if (base === undefined) {
- continue;
- }
- const value = queryValues.get(queryIdx) ?? penalty;
- ratios.push((10 + value) / (10 + base));
- }
- const score = geoMean(ratios);
- if (score === null) {
- continue;
- }
- rankings.push({ name, score, totalRuntime });
- }
- rankings.sort((a, b) =>
- a.score < b.score ? -1 : a.score > b.score ? 1 : compareCodeUnits(a.name, b.name),
+ const rankings = rankSeries(
+ rows.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })),
+ (a, b) => a - b,
+ QUERY_PENALTY_FLOOR_NS,
+ 0,
);
-
if (rankings.length === 0) {
return null;
}