diff --git a/docs/architecture/design-decisions.md b/docs/architecture/design-decisions.md
index bf06fe1..62e3f7d 100644
--- a/docs/architecture/design-decisions.md
+++ b/docs/architecture/design-decisions.md
@@ -146,6 +146,23 @@ slugs, and decoding validates the full shape, so they are not an injection
surface.
→ `web/lib/slug.ts`.
+**The random-access group summary aggregates the whole group, not its first chart.**
+The headline used to be lifted from the first chart in the group with data (in
+practice `feature-vectors/correlated`), which was fine when random access had one
+benchmark and misleading once it had many — the summary reported whatever that one
+dataset did. It now reports two aggregates per format over every dataset in the
+group: the **sum** and the **geomean** of their latest times, both labelled in the
+card. Aggregating means choosing a snapshot and a coverage rule: the aggregate is
+taken at the newest commit with any positive random-access row for the group (all
+of `random-access-bench`'s datasets and formats come from the same run), and a
+format is ranked only if it has a positive value for **every** dataset measured at
+that commit. That follows the compression summaries' "newest complete snapshot"
+precedent rather than the query summary's missing-series penalty, which works for
+ratios but would invent a runtime inside an absolute sum or geomean. The other
+groups' summaries are untouched — `RandomAccessGroup` is a singleton group with
+its own `collect*Summary`, so no other headline number moves.
+→ `web/lib/summary.ts` (`collectRandomAccessSummary`), `web/components/SummaryCard.tsx`.
+
## Performance & client hydration
The v4 stack is serverless and the site is low-traffic, so the costly case is
diff --git a/web/components/GroupSection.test.tsx b/web/components/GroupSection.test.tsx
index cd5c7de..c777ad1 100644
--- a/web/components/GroupSection.test.tsx
+++ b/web/components/GroupSection.test.tsx
@@ -23,7 +23,7 @@ const RANDOM_ACCESS: Group = {
summary: {
type: 'randomAccess',
title: 'Random Access Performance',
- rankings: [{ name: 'vortex', time: 1_500_000, ratio: 1 }],
+ rankings: [{ name: 'vortex', total: 3_000_000, geomean: 1_500_000, ratio: 1 }],
explanation: 'lower is better',
},
description: 'Tests selecting arbitrary row indices on NVMe',
diff --git a/web/components/SummaryCard.test.tsx b/web/components/SummaryCard.test.tsx
index a91e1fd..5045899 100644
--- a/web/components/SummaryCard.test.tsx
+++ b/web/components/SummaryCard.test.tsx
@@ -16,27 +16,34 @@ describe('SummaryCard', () => {
expect(render(undefined)).toBe('');
});
- it('renders a randomAccess card with ranks, ns times, and ratios', () => {
+ it('renders a randomAccess card with labelled geomean and total aggregates', () => {
const html = render({
type: 'randomAccess',
title: 'Random Access Performance',
rankings: [
- { name: 'vortex', time: 1_500_000, ratio: 1 },
- { name: 'parquet', time: 3_000_000, ratio: 2 },
+ { name: 'vortex', total: 6_000_000, geomean: 1_500_000, ratio: 1 },
+ { name: 'parquet', total: 12_000_000, geomean: 3_000_000, ratio: 2 },
],
- explanation: 'Random access time | Ratio to fastest (lower is better)',
+ explanation:
+ 'Geomean and total random access time across 4 datasets | ' +
+ 'Ratio of geomean to fastest (lower is better)',
});
expect(html).toContain('class="benchmark-scores-summary"');
expect(html).toContain('Random Access Performance
');
expect(html).toContain('#1');
expect(html).toContain('vortex');
- expect(html).toContain('1.50 ms');
+ // Both aggregates render, each labelled so the reader is not guessing which
+ // number is which.
+ expect(html).toContain('1.50 ms geomean');
+ expect(html).toContain('6.00 ms total');
expect(html).toContain('1.00x');
expect(html).toContain('#2');
expect(html).toContain('parquet');
- expect(html).toContain('3.00 ms');
+ expect(html).toContain('3.00 ms geomean');
+ expect(html).toContain('12.00 ms total');
expect(html).toContain('2.00x');
- expect(html).toContain('Random access time | Ratio to fastest (lower is better)');
+ expect(html).toContain('Geomean and total random access time across 4 datasets');
+ expect(html).toContain('Ratio of geomean to fastest (lower is better)');
});
it('renders nothing for a randomAccess card with no rankings', () => {
diff --git a/web/components/SummaryCard.tsx b/web/components/SummaryCard.tsx
index 3cd63c9..1fab99c 100644
--- a/web/components/SummaryCard.tsx
+++ b/web/components/SummaryCard.tsx
@@ -37,7 +37,8 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
{displaySeriesLabel(item.name)}
- {formatTimeNs(item.time)}
+ {formatTimeNs(item.geomean)} geomean
+ {formatTimeNs(item.total)} total
{item.ratio.toFixed(2)}x
diff --git a/web/lib/groups.test.ts b/web/lib/groups.test.ts
index 8c958de..a39ec5a 100644
--- a/web/lib/groups.test.ts
+++ b/web/lib/groups.test.ts
@@ -74,7 +74,7 @@ describe.skipIf(!dockerAvailable())(
]);
});
- it('computes the random-access summary (ratio to fastest)', async () => {
+ it('aggregates the random-access summary over the group (sum + geomean)', async () => {
const groups = await collectGroups();
const summary = expectDefined(
groups.find((g) => g.name === 'Random Access')?.summary,
@@ -84,10 +84,19 @@ describe.skipIf(!dockerAvailable())(
throw new Error(`expected randomAccess summary, got ${summary.type}`);
}
expect(summary.title).toBe('Random Access Performance');
+ expect(summary.explanation).toContain('across 1 dataset');
expect(summary.rankings[0].name).toBe('vortex-file-compressed');
expect(summary.rankings[1].name).toBe('parquet');
expect(summary.rankings[0].ratio).toBeCloseTo(1.0, 6);
expect(summary.rankings[1].ratio).toBeCloseTo(2.0, 6);
+ // The fixture's single `taxi` dataset at the newest commit: both
+ // aggregates collapse to that one latest value (500 / 1000 plus the
+ // third commit's bias), so the pre-aggregation headline is preserved
+ // for a one-chart group.
+ expect(summary.rankings[0].total).toBeCloseTo(100_500, 6);
+ expect(summary.rankings[0].geomean).toBeCloseTo(100_500, 6);
+ expect(summary.rankings[1].total).toBeCloseTo(201_000, 6);
+ expect(summary.rankings[1].geomean).toBeCloseTo(201_000, 6);
});
it('computes compression rankings with Parquet and Lance baselines', async () => {
diff --git a/web/lib/summary.test.ts b/web/lib/summary.test.ts
index 6310188..3545b52 100644
--- a/web/lib/summary.test.ts
+++ b/web/lib/summary.test.ts
@@ -67,3 +67,79 @@ describe('compression summaries', () => {
}
});
});
+
+describe('random-access summary', () => {
+ beforeEach(() => {
+ query.mockReset();
+ });
+
+ const CHARTS = [{ name: 'feature-vectors/correlated' }, { name: 'taxi' }];
+
+ it('aggregates every dataset in the group as a sum and a geomean', async () => {
+ query.mockResolvedValueOnce({
+ rows: [
+ { dataset: 'feature-vectors/correlated', name: 'vortex', value: 1_000_000 },
+ { dataset: 'feature-vectors/correlated', name: 'parquet', value: 4_000_000 },
+ { dataset: 'taxi', name: 'vortex', value: 4_000_000 },
+ { dataset: 'taxi', name: 'parquet', value: 16_000_000 },
+ ],
+ });
+
+ const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }, CHARTS);
+ if (summary === null || summary.type !== 'randomAccess') {
+ throw new Error('expected a randomAccess summary');
+ }
+ const byFormat = new Map(summary.rankings.map((ranking) => [ranking.name, ranking]));
+
+ // Both datasets contribute: sqrt(1e6 * 4e6) = 2e6, sum = 5e6.
+ expect(byFormat.get('vortex')?.geomean).toBeCloseTo(2_000_000, 6);
+ expect(byFormat.get('vortex')?.total).toBeCloseTo(5_000_000, 6);
+ expect(byFormat.get('parquet')?.geomean).toBeCloseTo(8_000_000, 6);
+ expect(byFormat.get('parquet')?.total).toBeCloseTo(20_000_000, 6);
+ // The ratio compares geomeans, not the first chart's raw value.
+ expect(byFormat.get('vortex')?.ratio).toBeCloseTo(1, 6);
+ expect(byFormat.get('parquet')?.ratio).toBeCloseTo(4, 6);
+ expect(summary.rankings.map((ranking) => ranking.name)).toEqual(['vortex', 'parquet']);
+ expect(summary.explanation).toBe(
+ 'Geomean and total random access time across 2 datasets | ' +
+ 'Ratio of geomean to fastest (lower is better)',
+ );
+ });
+
+ it("scopes one query to the group's datasets and its newest snapshot", async () => {
+ query.mockResolvedValue({ rows: [] });
+
+ expect(await collectGroupSummary({ k: 'RandomAccessGroup' }, CHARTS)).toBeNull();
+
+ expect(query).toHaveBeenCalledTimes(1);
+ const [text, params] = query.mock.calls[0] as [string, unknown[]];
+ expect(params).toEqual([['feature-vectors/correlated', 'taxi']]);
+ expect(text).toContain('r.dataset = ANY($1::text[])');
+ expect(text).toContain('SELECT MAX(ts) AS ts FROM scoped');
+ // A same-timestamp commit tie must not double-count into the sum.
+ expect(text).toContain('DISTINCT ON (s.dataset, s.format)');
+ });
+
+ it('drops a format that does not cover every measured dataset', async () => {
+ query.mockResolvedValueOnce({
+ rows: [
+ { dataset: 'feature-vectors/correlated', name: 'lance', value: 1_000 },
+ { dataset: 'feature-vectors/correlated', name: 'vortex', value: 1_000_000 },
+ { dataset: 'taxi', name: 'vortex', value: 4_000_000 },
+ ],
+ });
+
+ const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }, CHARTS);
+ if (summary === null || summary.type !== 'randomAccess') {
+ throw new Error('expected a randomAccess summary');
+ }
+ // Lance is missing `taxi` at the snapshot commit, so summing it over one
+ // dataset would make it look artificially cheap: it is dropped instead.
+ expect(summary.rankings.map((ranking) => ranking.name)).toEqual(['vortex']);
+ });
+
+ it('returns no summary for a group with no chart links', async () => {
+ expect(await collectGroupSummary({ k: 'RandomAccessGroup' }, [])).toBeNull();
+ expect(query).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/lib/summary.ts b/web/lib/summary.ts
index 30579bc..4d9c4aa 100644
--- a/web/lib/summary.ts
+++ b/web/lib/summary.ts
@@ -6,7 +6,8 @@
*
* Each `collect*Summary` runs focused SQL queries and returns one [`Summary`]
* variant. Query summaries use a v2 dataset allowlist. Compression summaries
- * compare the configured formats with Parquet.
+ * compare the configured formats with Parquet. The random-access summary
+ * aggregates every dataset in its group (sum + geomean).
*
* Behaviour-preservation notes (substrate migration, DuckDB -> Postgres):
* - Nullable-dim equality (`dataset_variant` / `scale_factor`) in the
@@ -55,13 +56,15 @@ function compressionSummaryQueryParams(): [string[], string[], string, string] {
];
}
-/** One random-access summary row. */
+/** One random-access summary row, aggregated over the whole group. */
export interface RandomAccessRanking {
/** Series name, normally the physical format. */
name: string;
- /** Latest measured time in nanoseconds. */
- time: number;
- /** Ratio to the fastest series in the same chart. */
+ /** Sum of the series' latest per-dataset times, in nanoseconds. */
+ total: number;
+ /** Geometric mean of the same per-dataset times, in nanoseconds. */
+ geomean: number;
+ /** Ratio of this series' geomean to the fastest series' geomean. */
ratio: number;
}
@@ -125,9 +128,9 @@ export type Summary =
/**
* Compute the summary for one group, if its kind has one.
- * `charts` is only consulted for the random-access path (which scans its chart
- * links for the latest populated dataset); the other paths query their fact
- * table directly. The structural `{ name }[]` accepts a `ChartLink[]`.
+ * `charts` is only consulted for the random-access path (whose chart links name
+ * the datasets it aggregates over); the other paths query their fact table
+ * directly. The structural `{ name }[]` accepts a `ChartLink[]`.
*/
export function collectGroupSummary(
key: GroupKey,
@@ -181,62 +184,135 @@ function geoMean(values: readonly number[]): number | null {
return n > 0 ? Math.exp(sumLn / n) : null;
}
+/**
+ * The random-access group summary: one row per format, aggregating **every**
+ * dataset in the group rather than reading a single chart.
+ *
+ * The group has grown from one random-access benchmark to many, so taking the
+ * headline from the first populated chart made the summary a report on whichever
+ * dataset happened to sort first (in practice `feature-vectors/correlated`, the
+ * noisiest one). Both aggregates are reported per format: the `total` (sum) and
+ * the `geomean` of the per-dataset times, the same pairing the query summary
+ * shows (`totalRuntime` + a geomean `score`).
+ *
+ * Two rules keep the two aggregates comparable across formats, following the
+ * compression summaries' "newest complete snapshot" precedent rather than the
+ * query summary's penalty imputation (which only works for ratios -- imputing
+ * into an absolute sum or geomean would invent a runtime):
+ *
+ * - **One snapshot commit.** All of `random-access-bench`'s formats and
+ * datasets are emitted by the same run, so the aggregate is taken at the
+ * newest commit that has any positive random-access row for this group.
+ * - **Complete coverage or nothing.** A format is ranked only if it has a
+ * positive value for every dataset measured at that commit; a format missing
+ * a dataset is dropped instead of being summed over a smaller (and therefore
+ * flatteringly cheaper) set. Non-positive and absent values never enter the
+ * geomean, which is defined over strictly positive values only.
+ */
async function collectRandomAccessSummary(
charts: readonly { readonly name: string }[],
): Promise {
- // Scan the group's chart links in order; the first chart with valid rows at
- // its latest commit wins (matching the Rust early-return loop).
+ // The group's chart names ARE its dataset names (`collectRandomAccessGroup`
+ // builds one chart link per distinct dataset), so they scope the aggregate to
+ // this group's results in a single query instead of the previous
+ // one-query-per-chart-until-a-hit loop.
+ const datasets = charts.map((chart) => chart.name);
+ if (datasets.length === 0) {
+ return null;
+ }
const text = `
- SELECT r.format AS name, r.value_ns::float8 AS value
- FROM random_access_times r
- JOIN commits c USING (commit_sha)
- WHERE r.dataset = $1
- AND r.value_ns > 0
- AND c.timestamp = (
- SELECT MAX(c2.timestamp)
- FROM random_access_times r2
- JOIN commits c2 USING (commit_sha)
- WHERE r2.dataset = $2
- AND r2.value_ns > 0
- )
- ORDER BY r.value_ns, r.format
+ WITH scoped AS (
+ SELECT r.dataset AS dataset,
+ r.format AS format,
+ r.commit_sha AS commit_sha,
+ c.timestamp AS ts,
+ r.value_ns::float8 AS value
+ FROM random_access_times r
+ JOIN commits c USING (commit_sha)
+ WHERE r.dataset = ANY($1::text[])
+ AND r.value_ns > 0
+ ), snapshot AS (
+ SELECT MAX(ts) AS ts FROM scoped
+ )
+ SELECT DISTINCT ON (s.dataset, s.format)
+ s.dataset AS dataset, s.format AS name, s.value AS value
+ FROM scoped s
+ JOIN snapshot ON s.ts = snapshot.ts
+ ORDER BY s.dataset, s.format, s.commit_sha
`;
- for (const chart of charts) {
- const rows = (
- await getPool().query<{ name: string; value: number }>(text, [chart.name, chart.name])
- ).rows;
- const rankings: RandomAccessRanking[] = rows.map((row) => ({
- name: row.name,
- time: row.value,
- ratio: 0,
- }));
- if (rankings.length === 0) {
- continue;
+ // `DISTINCT ON` collapses a same-timestamp commit tie (the accepted
+ // same-second tie behaviour elsewhere in the read path) to one row per
+ // (dataset, format), so a tie cannot double-count into the sum.
+ const rows = (
+ await getPool().query<{ dataset: string; name: string; value: number }>(text, [datasets])
+ ).rows;
+ const measured = new Set();
+ const valuesByFormat = new Map>();
+ for (const row of rows) {
+ measured.add(row.dataset);
+ let byDataset = valuesByFormat.get(row.name);
+ if (byDataset === undefined) {
+ byDataset = new Map();
+ valuesByFormat.set(row.name, byDataset);
}
- // Streaming min (loop, not a `Math.min(...)` spread) for consistency with
- // `collectCompressionSizeSummary` and to avoid a large-array call-argument
- // cliff; the Rust source uses `reduce(f64::min)` here.
- let minTime = Infinity;
- for (const r of rankings) {
- minTime = Math.min(minTime, r.time);
+ byDataset.set(row.dataset, row.value);
+ }
+ if (measured.size === 0) {
+ return null;
+ }
+
+ const rankings: RandomAccessRanking[] = [];
+ for (const name of [...valuesByFormat.keys()].sort(compareCodeUnits)) {
+ const byDataset = valuesByFormat.get(name);
+ if (byDataset === undefined || byDataset.size !== measured.size) {
+ // Incomplete coverage of the snapshot's datasets: drop the format rather
+ // than publish a sum and geomean over a different dataset set.
+ continue;
}
- if (minTime <= 0 || !Number.isFinite(minTime)) {
+ // The size check above means this format's dataset keys are exactly
+ // `measured`, so the `?? 0` is unreachable; a 0 would be excluded from the
+ // geomean anyway.
+ const values = [...measured]
+ .sort(compareCodeUnits)
+ .map((dataset) => byDataset.get(dataset) ?? 0);
+ const geomean = geoMean(values);
+ if (geomean === null) {
continue;
}
- for (const r of rankings) {
- r.ratio = r.time / minTime;
+ let total = 0;
+ for (const value of values) {
+ total += value;
}
- rankings.sort((a, b) =>
- a.time < b.time ? -1 : a.time > b.time ? 1 : compareCodeUnits(a.name, b.name),
- );
- return {
- type: 'randomAccess',
- title: 'Random Access Performance',
- rankings,
- explanation: 'Random access time | Ratio to fastest (lower is better)',
- };
+ rankings.push({ name, total, geomean, ratio: 0 });
+ }
+ if (rankings.length === 0) {
+ return null;
}
- return null;
+ // Streaming min (loop, not a `Math.min(...)` spread) for consistency with
+ // `collectCompressionSizeSummary` and to avoid a large-array call-argument
+ // cliff.
+ let fastest = Infinity;
+ for (const ranking of rankings) {
+ fastest = Math.min(fastest, ranking.geomean);
+ }
+ if (fastest <= 0 || !Number.isFinite(fastest)) {
+ return null;
+ }
+ for (const ranking of rankings) {
+ ranking.ratio = ranking.geomean / fastest;
+ }
+ rankings.sort((a, b) =>
+ a.geomean < b.geomean ? -1 : a.geomean > b.geomean ? 1 : compareCodeUnits(a.name, b.name),
+ );
+ const datasetNoun = measured.size === 1 ? 'dataset' : 'datasets';
+ return {
+ type: 'randomAccess',
+ title: 'Random Access Performance',
+ rankings,
+ explanation:
+ `Geomean and total random access time across ${measured.size} ${datasetNoun} | ` +
+ 'Ratio of geomean to fastest (lower is better)',
+ };
}
async function collectCompressionSummary(): Promise {