Aggregate the random-access group summary over every dataset

docs/architecture/design-decisions.md

@@ -146,6 +146,23 @@ slugs, and decoding validates the full shape, so they are not an injection
 surface.
 → `web/lib/slug.ts`.
 
+**The random-access group summary aggregates the whole group, not its first chart.**
+The headline used to be lifted from the first chart in the group with data (in
+practice `feature-vectors/correlated`), which was fine when random access had one
+benchmark and misleading once it had many — the summary reported whatever that one
+dataset did. It now reports two aggregates per format over every dataset in the
+group: the **sum** and the **geomean** of their latest times, both labelled in the
+card. Aggregating means choosing a snapshot and a coverage rule: the aggregate is
+taken at the newest commit with any positive random-access row for the group (all
+of `random-access-bench`'s datasets and formats come from the same run), and a
+format is ranked only if it has a positive value for **every** dataset measured at
+that commit. That follows the compression summaries' "newest complete snapshot"
+precedent rather than the query summary's missing-series penalty, which works for
+ratios but would invent a runtime inside an absolute sum or geomean. The other
+groups' summaries are untouched — `RandomAccessGroup` is a singleton group with
+its own `collect*Summary`, so no other headline number moves.
+→ `web/lib/summary.ts` (`collectRandomAccessSummary`), `web/components/SummaryCard.tsx`.
+
 ## Performance & client hydration
 
 The v4 stack is serverless and the site is low-traffic, so the costly case is

web/components/GroupSection.test.tsx

@@ -23,7 +23,7 @@ const RANDOM_ACCESS: Group = {
   summary: {
     type: 'randomAccess',
     title: 'Random Access Performance',
-    rankings: [{ name: 'vortex', time: 1_500_000, ratio: 1 }],
+    rankings: [{ name: 'vortex', total: 3_000_000, geomean: 1_500_000, ratio: 1 }],
     explanation: 'lower is better',
   },
   description: 'Tests selecting arbitrary row indices on NVMe',

web/components/SummaryCard.test.tsx

@@ -16,27 +16,34 @@ describe('SummaryCard', () => {
     expect(render(undefined)).toBe('');
   });
 
-  it('renders a randomAccess card with ranks, ns times, and ratios', () => {
+  it('renders a randomAccess card with labelled geomean and total aggregates', () => {
     const html = render({
       type: 'randomAccess',
       title: 'Random Access Performance',
       rankings: [
-        { name: 'vortex', time: 1_500_000, ratio: 1 },
-        { name: 'parquet', time: 3_000_000, ratio: 2 },
+        { name: 'vortex', total: 6_000_000, geomean: 1_500_000, ratio: 1 },
+        { name: 'parquet', total: 12_000_000, geomean: 3_000_000, ratio: 2 },
       ],
-      explanation: 'Random access time | Ratio to fastest (lower is better)',
+      explanation:
+        'Geomean and total random access time across 4 datasets | ' +
+        'Ratio of geomean to fastest (lower is better)',
     });
     expect(html).toContain('class="benchmark-scores-summary"');
     expect(html).toContain('<h3 class="scores-title">Random Access Performance</h3>');
     expect(html).toContain('#1');
     expect(html).toContain('vortex');
-    expect(html).toContain('1.50 ms');
+    // Both aggregates render, each labelled so the reader is not guessing which
+    // number is which.
+    expect(html).toContain('1.50 ms geomean');
+    expect(html).toContain('6.00 ms total');
     expect(html).toContain('1.00x');
     expect(html).toContain('#2');
     expect(html).toContain('parquet');
-    expect(html).toContain('3.00 ms');
+    expect(html).toContain('3.00 ms geomean');
+    expect(html).toContain('12.00 ms total');
     expect(html).toContain('2.00x');
-    expect(html).toContain('Random access time | Ratio to fastest (lower is better)');
+    expect(html).toContain('Geomean and total random access time across 4 datasets');
+    expect(html).toContain('Ratio of geomean to fastest (lower is better)');
   });
 
   it('renders nothing for a randomAccess card with no rankings', () => {

web/components/SummaryCard.tsx

@@ -37,7 +37,8 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
                   {displaySeriesLabel(item.name)}
                 </span>
                 <span className="score-metrics">
-                  <span className="score-value">{formatTimeNs(item.time)}</span>
+                  <span className="score-value">{formatTimeNs(item.geomean)} geomean</span>
+                  <span className="score-runtime">{formatTimeNs(item.total)} total</span>
                   <span className="score-runtime">{item.ratio.toFixed(2)}x</span>
                 </span>
               </div>

web/lib/groups.test.ts

@@ -74,7 +74,7 @@ describe.skipIf(!dockerAvailable())(
       ]);
     });
 
-    it('computes the random-access summary (ratio to fastest)', async () => {
+    it('aggregates the random-access summary over the group (sum + geomean)', async () => {
       const groups = await collectGroups();
       const summary = expectDefined(
         groups.find((g) => g.name === 'Random Access')?.summary,
@@ -84,10 +84,19 @@ describe.skipIf(!dockerAvailable())(
         throw new Error(`expected randomAccess summary, got ${summary.type}`);
       }
       expect(summary.title).toBe('Random Access Performance');
+      expect(summary.explanation).toContain('across 1 dataset');
       expect(summary.rankings[0].name).toBe('vortex-file-compressed');
       expect(summary.rankings[1].name).toBe('parquet');
       expect(summary.rankings[0].ratio).toBeCloseTo(1.0, 6);
       expect(summary.rankings[1].ratio).toBeCloseTo(2.0, 6);
+      // The fixture's single `taxi` dataset at the newest commit: both
+      // aggregates collapse to that one latest value (500 / 1000 plus the
+      // third commit's bias), so the pre-aggregation headline is preserved
+      // for a one-chart group.
+      expect(summary.rankings[0].total).toBeCloseTo(100_500, 6);
+      expect(summary.rankings[0].geomean).toBeCloseTo(100_500, 6);
+      expect(summary.rankings[1].total).toBeCloseTo(201_000, 6);
+      expect(summary.rankings[1].geomean).toBeCloseTo(201_000, 6);
     });
 
     it('computes compression rankings with Parquet and Lance baselines', async () => {

web/lib/summary.test.ts

@@ -67,3 +67,79 @@ describe('compression summaries', () => {
     }
   });
 });
+
+describe('random-access summary', () => {
+  beforeEach(() => {
+    query.mockReset();
+  });
+
+  const CHARTS = [{ name: 'feature-vectors/correlated' }, { name: 'taxi' }];
+
+  it('aggregates every dataset in the group as a sum and a geomean', async () => {
+    query.mockResolvedValueOnce({
+      rows: [
+        { dataset: 'feature-vectors/correlated', name: 'vortex', value: 1_000_000 },
+        { dataset: 'feature-vectors/correlated', name: 'parquet', value: 4_000_000 },
+        { dataset: 'taxi', name: 'vortex', value: 4_000_000 },
+        { dataset: 'taxi', name: 'parquet', value: 16_000_000 },
+      ],
+    });
+
+    const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }, CHARTS);
+    if (summary === null || summary.type !== 'randomAccess') {
+      throw new Error('expected a randomAccess summary');
+    }
+    const byFormat = new Map(summary.rankings.map((ranking) => [ranking.name, ranking]));
+
+    // Both datasets contribute: sqrt(1e6 * 4e6) = 2e6, sum = 5e6.
+    expect(byFormat.get('vortex')?.geomean).toBeCloseTo(2_000_000, 6);
+    expect(byFormat.get('vortex')?.total).toBeCloseTo(5_000_000, 6);
+    expect(byFormat.get('parquet')?.geomean).toBeCloseTo(8_000_000, 6);
+    expect(byFormat.get('parquet')?.total).toBeCloseTo(20_000_000, 6);
+    // The ratio compares geomeans, not the first chart's raw value.
+    expect(byFormat.get('vortex')?.ratio).toBeCloseTo(1, 6);
+    expect(byFormat.get('parquet')?.ratio).toBeCloseTo(4, 6);
+    expect(summary.rankings.map((ranking) => ranking.name)).toEqual(['vortex', 'parquet']);
+    expect(summary.explanation).toBe(
+      'Geomean and total random access time across 2 datasets | ' +
+        'Ratio of geomean to fastest (lower is better)',
+    );
+  });
+
+  it("scopes one query to the group's datasets and its newest snapshot", async () => {
+    query.mockResolvedValue({ rows: [] });
+
+    expect(await collectGroupSummary({ k: 'RandomAccessGroup' }, CHARTS)).toBeNull();
+
+    expect(query).toHaveBeenCalledTimes(1);
+    const [text, params] = query.mock.calls[0] as [string, unknown[]];
+    expect(params).toEqual([['feature-vectors/correlated', 'taxi']]);
+    expect(text).toContain('r.dataset = ANY($1::text[])');
+    expect(text).toContain('SELECT MAX(ts) AS ts FROM scoped');
+    // A same-timestamp commit tie must not double-count into the sum.
+    expect(text).toContain('DISTINCT ON (s.dataset, s.format)');
+  });
+
+  it('drops a format that does not cover every measured dataset', async () => {
+    query.mockResolvedValueOnce({
+      rows: [
+        { dataset: 'feature-vectors/correlated', name: 'lance', value: 1_000 },
+        { dataset: 'feature-vectors/correlated', name: 'vortex', value: 1_000_000 },
+        { dataset: 'taxi', name: 'vortex', value: 4_000_000 },
+      ],
+    });
+
+    const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }, CHARTS);
+    if (summary === null || summary.type !== 'randomAccess') {
+      throw new Error('expected a randomAccess summary');
+    }
+    // Lance is missing `taxi` at the snapshot commit, so summing it over one
+    // dataset would make it look artificially cheap: it is dropped instead.
+    expect(summary.rankings.map((ranking) => ranking.name)).toEqual(['vortex']);
+  });
+
+  it('returns no summary for a group with no chart links', async () => {
+    expect(await collectGroupSummary({ k: 'RandomAccessGroup' }, [])).toBeNull();
+    expect(query).not.toHaveBeenCalled();
+  });
+});