diff --git a/.changeset/html-parallel-minify.md b/.changeset/html-parallel-minify.md new file mode 100644 index 000000000..aea4284c9 --- /dev/null +++ b/.changeset/html-parallel-minify.md @@ -0,0 +1,5 @@ +--- +'@doc-kit/generator-react': minor +--- + +perf(html): one client entry chunk, HTML minification in the worker pool diff --git a/.changeset/section-pages-generator.md b/.changeset/section-pages-generator.md new file mode 100644 index 000000000..a435ef875 --- /dev/null +++ b/.changeset/section-pages-generator.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +feat: `dependent` generators and the `section-pages` generator diff --git a/docs/creating-generators.md b/docs/creating-generators.md index 6580fbbc9..2842a6458 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -411,6 +411,63 @@ export async function generate(input, worker) { } ``` +### Declaring a dependent + +`dependsOn` pulls another generator's output _in_. The inverse, `dependent`, +pushes a generator's output _into_ another generator's pipeline: + +```mjs displayName="index.mjs" +export default { + name: 'section-pages', + + dependsOn: '@doc-kit/core/metadata', + + // Deliver this generator's output through `html` + dependent: '@doc-kit/generator-react/html', + + generate, +}; +``` + +The pipeline splices such a generator in front of the first generator on the +way to its dependent that consumes the same `dependsOn`. Here `html` depends on +`jsx-ast`, which depends on `metadata` — so `jsx-ast` is rewired to read from +`section-pages` instead: + +```text +ast → metadata → section-pages → jsx-ast → html +``` + +Requesting a generator that declares a dependent runs the dependent's whole +pipeline, and the run's result is the dependent's output — `-t section-pages` +produces the `html` site. Several generators may splice in at the same point; +they form a chain. A generator whose dependent pipeline never consumes its +`dependsOn` is an error. + +`dependent` also accepts an array. The generator is spliced into every listed +pipeline; when requested, it is delivered through the dependents that are +already part of the run, or through all of them when none is: + +```mjs displayName="index.mjs" +export default { + name: 'section-pages', + dependsOn: '@doc-kit/core/metadata', + dependent: [ + '@doc-kit/generator-react/html', + '@doc-kit/generator-react/sitemap', + ], + generate, +}; +``` + +With this, `-t section-pages` builds the site and the sitemap, while +`-t html -t section-pages` builds just the site. + +Use a dependent when a generator transforms an intermediate representation +(adding, filtering, or rewriting entries) rather than producing a new output +format of its own. See the [`section-pages`](./generators/section-pages.md) generator for +a worked example. + ## File Output ### Writing Output Files diff --git a/docs/generators.md b/docs/generators.md index dd5536664..729864a23 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -10,12 +10,13 @@ npx @doc-kit/cli generate -t html -t orama-db -t sitemap -i "docs/**/*.md" -o ou ### Web ([`@doc-kit/generator-react`](./packages/react.md)) -| Target | Output | -| -------------------------------------- | -------------------------------------------------------------------- | -| [`html`](./generators/html.md) | The modern documentation site: server-rendered, hydrated, themeable. | -| [`orama-db`](./generators/orama-db.md) | The search index behind the `html` site's search box. | -| [`llms-txt`](./generators/llms-txt.md) | An [`llms.txt`](https://llmstxt.org/) index for language models. | -| [`sitemap`](./generators/sitemap.md) | A `sitemap.xml` for search engines. | +| Target | Output | +| ------------------------------------------------ | -------------------------------------------------------------------- | +| [`html`](./generators/html.md) | The modern documentation site: server-rendered, hydrated, themeable. | +| [`orama-db`](./generators/orama-db.md) | The search index behind the `html` site's search box. | +| [`llms-txt`](./generators/llms-txt.md) | An [`llms.txt`](https://llmstxt.org/) index for language models. | +| [`sitemap`](./generators/sitemap.md) | A `sitemap.xml` for search engines. | +| [`section-pages`](./generators/section-pages.md) | The `html` site and sitemap, plus one page per section of a module. | ### JSON ([`@doc-kit/core`](./packages/core.md)) diff --git a/packages/core/src/__tests__/generators.test.mjs b/packages/core/src/__tests__/generators.test.mjs index 40af019f2..c4dff83bb 100644 --- a/packages/core/src/__tests__/generators.test.mjs +++ b/packages/core/src/__tests__/generators.test.mjs @@ -72,6 +72,33 @@ const syntheticGenerators = { return { all: input }; }, }, + // A generator delivered through another pipeline: it reads `metadata` and + // declares `gen-d-all` as its dependent, so `gen-d` reads from it instead. + 'gen-splice': { + name: 'gen-splice', + dependsOn: 'metadata', + dependent: 'gen-d-all', + generate: async input => { + record('gen-splice'); + return [...input, { spliced: true }]; + }, + }, + 'gen-d': { + name: 'gen-d', + dependsOn: 'metadata', + generate: async input => { + record('gen-d'); + return { d: input }; + }, + }, + 'gen-d-all': { + name: 'gen-d-all', + dependsOn: 'gen-d', + generate: async input => { + record('gen-d-all'); + return { all: input }; + }, + }, }; mock.module('../generators/loader.mjs', { @@ -92,8 +119,10 @@ mock.module('../generators/loader.mjs', { const generator = syntheticGenerators[specifier]; generators.set(specifier, generator); - if (generator.dependsOn) { - queue.push(generator.dependsOn); + for (const related of [generator.dependsOn, generator.dependent]) { + if (related) { + queue.push(related); + } } } @@ -162,4 +191,24 @@ describe('createGenerator orchestration', () => { assert.deepStrictEqual(results, [[{ c: true }], { all: [{ c: true }] }]); }); + + it('delivers a generator through its dependent', async () => { + const { runGenerators } = createGenerator(); + + const results = await runGenerators({ + target: ['gen-splice'], + threads: 1, + }); + + // Requesting only `gen-splice` runs its dependent's whole pipeline, with + // `gen-d` reading the spliced output instead of `metadata` directly. + assert.equal(runs.metadata, 1); + assert.equal(runs['gen-splice'], 1); + assert.equal(runs['gen-d'], 1); + assert.equal(runs['gen-d-all'], 1); + + assert.deepStrictEqual(results, [ + { all: { d: [{ meta: 1 }, { spliced: true }] } }, + ]); + }); }); diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index b079dc0d0..79a61235e 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -5,6 +5,7 @@ import { loadGenerators, resolveGeneratorSpecifier, } from './generators/loader.mjs'; +import { resolvePipeline } from './generators/pipeline.mjs'; import logger from './logger/index.mjs'; import createWorkerPool from './threading/index.mjs'; import createParallelWorker from './threading/parallel.mjs'; @@ -44,9 +45,12 @@ const createGenerator = () => { * * @param {string} specifier - Resolved generator specifier to schedule * @param {Map} generators - Loaded generators + * @param {Map} inputOf - Each generator's + * effective input generator (its dependency, unless another generator was + * spliced in front of it via `dependent`) * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options */ - const scheduleGenerator = (specifier, generators, configuration) => { + const scheduleGenerator = (specifier, generators, inputOf, configuration) => { if (cache.has(specifier)) { return; } @@ -54,12 +58,11 @@ const createGenerator = () => { const generator = generators.get(specifier); const { name, generate, hasParallelProcessor } = generator; - const dependsOn = - generator.dependsOn && resolveGeneratorSpecifier(generator.dependsOn); + const dependsOn = inputOf.get(specifier); // Schedule dependency first if (dependsOn && !cache.has(dependsOn)) { - scheduleGenerator(dependsOn, generators, configuration); + scheduleGenerator(dependsOn, generators, inputOf, configuration); } generatorsLogger.debug(`Scheduling "${name}"`, { @@ -104,8 +107,16 @@ const createGenerator = () => { // Resolve shorthand names and load the full dependency closure up front, // so scheduling below is fully synchronous. - const targets = target.map(resolveGeneratorSpecifier); - const generators = await loadGenerators(targets); + const generators = await loadGenerators( + target.map(resolveGeneratorSpecifier) + ); + + // Work out who reads from whom once generators declaring a `dependent` + // have been spliced in, and which generators the run finally collects. + const { targets, inputOf } = resolvePipeline( + target.map(resolveGeneratorSpecifier), + generators + ); generatorsLogger.debug(`Starting pipeline`, { generators: targets.join(', '), @@ -114,18 +125,14 @@ const createGenerator = () => { // Compute consumer counts up front so dependencies can be evicted as soon // as their last consumer runs (must be ready before any generator starts). - cache.populateConsumerCounts(targets, specifier => { - const { dependsOn } = generators.get(specifier); - - return dependsOn && resolveGeneratorSpecifier(dependsOn); - }); + cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier)); // Create worker pool pool = createWorkerPool(threads); // Schedule all generators for (const specifier of targets) { - scheduleGenerator(specifier, generators, configuration); + scheduleGenerator(specifier, generators, inputOf, configuration); } // Start all collections in parallel (don't await sequentially). Consuming diff --git a/packages/core/src/generators/__tests__/index.test.mjs b/packages/core/src/generators/__tests__/index.test.mjs index e0af86704..7f1bde1a8 100644 --- a/packages/core/src/generators/__tests__/index.test.mjs +++ b/packages/core/src/generators/__tests__/index.test.mjs @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { enforceArray } from '#utils/array.mjs'; + import { allGenerators, deprecatedGenerators, @@ -62,6 +64,17 @@ describe('All Generators', () => { }); }); + it('should have valid dependent references', () => { + loadedGenerators.forEach(([name, , generator]) => { + for (const dependent of enforceArray(generator.dependent ?? [])) { + assert.ok( + validDependencies.includes(dependent), + `Generator "${name}" declares dependent "${dependent}" which is not a valid generator specifier` + ); + } + }); + }); + it('should resolve deprecated aliases to loadable generators', async () => { for (const [name, specifier] of Object.entries(deprecatedGenerators)) { assert.equal(resolveGeneratorSpecifier(name), specifier); diff --git a/packages/core/src/generators/__tests__/pipeline.test.mjs b/packages/core/src/generators/__tests__/pipeline.test.mjs new file mode 100644 index 000000000..dd9a66bc7 --- /dev/null +++ b/packages/core/src/generators/__tests__/pipeline.test.mjs @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { resolvePipeline } from '../pipeline.mjs'; + +// Specifiers are deliberately not shorthand names, so they resolve to +// themselves. +const generators = definitions => + new Map( + Object.entries(definitions).map(([name, generator]) => [ + name, + { name, ...generator }, + ]) + ); + +const web = { + source: {}, + parse: { dependsOn: 'source' }, + render: { dependsOn: 'parse' }, + site: { dependsOn: 'render' }, +}; + +describe('resolvePipeline', () => { + it('reads each generator from its dependency by default', () => { + const { targets, inputOf } = resolvePipeline(['site'], generators(web)); + + assert.deepEqual(targets, ['site']); + assert.deepEqual( + [...inputOf], + [ + ['source', undefined], + ['parse', 'source'], + ['render', 'parse'], + ['site', 'render'], + ] + ); + }); + + it('splices a generator in front of the consumer of its own input', () => { + const { targets, inputOf } = resolvePipeline( + ['splice'], + generators({ + ...web, + splice: { dependsOn: 'parse', dependent: 'site' }, + }) + ); + + // The run delivers `splice` through `site` + assert.deepEqual(targets, ['site']); + assert.equal(inputOf.get('splice'), 'parse'); + assert.equal(inputOf.get('render'), 'splice'); + assert.equal(inputOf.get('site'), 'render'); + }); + + it('collects a dependent only once when it is also requested', () => { + const { targets } = resolvePipeline( + ['site', 'splice'], + generators({ + ...web, + splice: { dependsOn: 'parse', dependent: 'site' }, + }) + ); + + assert.deepEqual(targets, ['site']); + }); + + it('chains several generators splicing at the same point', () => { + const { inputOf } = resolvePipeline( + ['first', 'second'], + generators({ + ...web, + first: { dependsOn: 'parse', dependent: 'site' }, + second: { dependsOn: 'parse', dependent: 'site' }, + }) + ); + + assert.equal(inputOf.get('render'), 'first'); + assert.equal(inputOf.get('first'), 'second'); + assert.equal(inputOf.get('second'), 'parse'); + }); + + it('can splice a source generator in front of the pipeline root', () => { + const { inputOf } = resolvePipeline( + ['feed'], + generators({ ...web, feed: { dependent: 'site' } }) + ); + + assert.equal(inputOf.get('source'), 'feed'); + assert.equal(inputOf.get('feed'), undefined); + }); + + it('leaves unrelated pipelines untouched', () => { + const { inputOf } = resolvePipeline( + ['splice', 'other'], + generators({ + ...web, + other: { dependsOn: 'parse' }, + splice: { dependsOn: 'parse', dependent: 'site' }, + }) + ); + + assert.equal(inputOf.get('other'), 'parse'); + }); + + it('delivers through every dependent when none is part of the run', () => { + const { targets, inputOf } = resolvePipeline( + ['splice'], + generators({ + ...web, + map: { dependsOn: 'parse' }, + splice: { dependsOn: 'parse', dependent: ['site', 'map'] }, + }) + ); + + assert.deepEqual(targets, ['site', 'map']); + assert.equal(inputOf.get('render'), 'splice'); + assert.equal(inputOf.get('map'), 'splice'); + }); + + it('delivers only through the dependents already part of the run', () => { + const { targets, inputOf } = resolvePipeline( + ['site', 'splice'], + generators({ + ...web, + map: { dependsOn: 'parse' }, + splice: { dependsOn: 'parse', dependent: ['site', 'map'] }, + }) + ); + + assert.deepEqual(targets, ['site']); + // The splice still applies to `map`, should anything run it + assert.equal(inputOf.get('map'), 'splice'); + }); + + it('throws when the dependent pipeline never consumes the input', () => { + assert.throws( + () => + resolvePipeline( + ['splice'], + generators({ + ...web, + aside: {}, + splice: { dependsOn: 'aside', dependent: 'site' }, + }) + ), + /nothing in that pipeline consumes "aside"/ + ); + }); +}); diff --git a/packages/core/src/generators/index.mjs b/packages/core/src/generators/index.mjs index e58f3aeb7..5acae57d9 100644 --- a/packages/core/src/generators/index.mjs +++ b/packages/core/src/generators/index.mjs @@ -22,6 +22,7 @@ export const publicGenerators = { 'llms-txt': '@doc-kit/generator-react/llms-txt', sitemap: '@doc-kit/generator-react/sitemap', html: '@doc-kit/generator-react/html', + 'section-pages': '@doc-kit/generator-react/section-pages', }; // These ones are special since they don't produce standard output, diff --git a/packages/core/src/generators/loader.mjs b/packages/core/src/generators/loader.mjs index 653d202bc..49ba85b53 100644 --- a/packages/core/src/generators/loader.mjs +++ b/packages/core/src/generators/loader.mjs @@ -3,6 +3,8 @@ import { isAbsolute } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { enforceArray } from '#utils/array.mjs'; + import { allGenerators } from './index.mjs'; /** @@ -67,7 +69,7 @@ export const loadGenerator = async specifier => { /** * Loads the given generators plus the transitive closure of their - * dependencies (via `dependsOn`). + * dependencies (via `dependsOn`) and dependents (via `dependent`). * * @param {string[]} targets - Shorthand names, paths, or import specifiers * @returns {Promise>} Loaded generators keyed @@ -87,8 +89,11 @@ export const loadGenerators = async targets => { const generator = await loadGenerator(specifier); generators.set(specifier, generator); - if (generator.dependsOn) { - queue.push(resolveGeneratorSpecifier(generator.dependsOn)); + for (const related of [ + ...enforceArray(generator.dependsOn ?? []), + ...enforceArray(generator.dependent ?? []), + ]) { + queue.push(resolveGeneratorSpecifier(related)); } } diff --git a/packages/core/src/generators/metadata/types.d.ts b/packages/core/src/generators/metadata/types.d.ts index 4789a7aab..6be3d2e5d 100644 --- a/packages/core/src/generators/metadata/types.d.ts +++ b/packages/core/src/generators/metadata/types.d.ts @@ -120,6 +120,23 @@ export interface StabilityData extends Data { */ export type StabilityNode = NodeWithData; +/** + * Describes a chunk page: a section of a module rendered as a page of its own + * (see the `section-pages` generator). Present on every entry of a chunk page. + */ +export interface ChunkInfo { + /** The `api` of the module the chunk was split from */ + api: string; + /** The `path` of the module the chunk was split from */ + path: string; + /** The section's anchor on the full module page */ + slug: string; + /** The chunk's position within the module, in document order */ + index: number; + /** The original depth of the chunk's heading */ + depth: number; +} + /** * Complete metadata entry for a single API documentation element. * @@ -136,6 +153,8 @@ export interface MetadataEntry extends YAMLProperties { synthetic?: boolean; /** Whether this page was authored as MDX (JSX-in-Markdown). */ mdx?: boolean; + /** Set when this entry belongs to a chunk page rather than a full page. */ + chunk?: ChunkInfo; /** Processed heading with metadata */ heading: HeadingNode; /** Stability classification information */ diff --git a/packages/core/src/generators/pipeline.mjs b/packages/core/src/generators/pipeline.mjs new file mode 100644 index 000000000..933fe23b3 --- /dev/null +++ b/packages/core/src/generators/pipeline.mjs @@ -0,0 +1,169 @@ +'use strict'; + +import { enforceArray } from '#utils/array.mjs'; + +import { resolveGeneratorSpecifier } from './loader.mjs'; + +/** + * The resolved specifiers of the generators a generator is delivered through. + * + * @param {GeneratorMetadata | undefined} generator + * @returns {string[]} + */ +const dependentsOf = generator => + enforceArray(generator?.dependent ?? []).map(resolveGeneratorSpecifier); + +/** + * Collects the generators reached from the given ones through `dependsOn`. + * + * @param {string[]} specifiers - Resolved generator specifiers + * @param {Map} generators - Loaded generators + * @returns {Set} + */ +const dependencyClosure = (specifiers, generators) => { + const closure = new Set(); + const stack = [...specifiers]; + + while (stack.length > 0) { + const specifier = stack.pop(); + + if (closure.has(specifier)) { + continue; + } + + closure.add(specifier); + + const { dependsOn } = generators.get(specifier) ?? {}; + + if (dependsOn) { + stack.push(resolveGeneratorSpecifier(dependsOn)); + } + } + + return closure; +}; + +/** + * Splices a generator into a dependent's pipeline: walking from the dependent + * up its dependency chain, the first generator that consumes the splicing + * generator's own input is rewired to consume the splicing generator instead. + * + * @param {string} specifier - The generator to splice in + * @param {string} dependent - The pipeline to splice it into + * @param {Map} inputOf - Effective inputs, mutated + * @param {string} name - The splicing generator's name, for errors + */ +const splice = (specifier, dependent, inputOf, name) => { + const upstream = inputOf.get(specifier); + const visited = new Set(); + + let current = dependent; + + while (current !== undefined && current !== specifier) { + if (visited.has(current)) { + break; + } + + visited.add(current); + + if (inputOf.get(current) === upstream) { + inputOf.set(current, specifier); + return; + } + + current = inputOf.get(current); + } + + throw new Error( + `Generator "${name}" declares "${dependent}" as its dependent, but ` + + `nothing in that pipeline consumes "${upstream ?? 'no input'}", so ` + + 'there is nowhere to splice it in.' + ); +}; + +/** + * Follows a generator's `dependent` links to the generators that finally + * deliver its output. When some of its dependents are already part of the + * run, only those deliver it; otherwise all of them do. + * + * @param {string} specifier - Resolved generator specifier + * @param {Map} generators - Loaded generators + * @param {Set} requested - Generators already part of the run + * @param {Set} [seen] - Guards against dependent cycles + * @returns {string[]} Resolved specifiers of the terminal generators + */ +const resolveTerminals = ( + specifier, + generators, + requested, + seen = new Set() +) => { + if (seen.has(specifier)) { + return []; + } + + seen.add(specifier); + + const dependents = dependentsOf(generators.get(specifier)); + + if (dependents.length === 0) { + return [specifier]; + } + + const active = dependents.filter(dependent => requested.has(dependent)); + + return (active.length > 0 ? active : dependents).flatMap(dependent => + resolveTerminals(dependent, generators, requested, seen) + ); +}; + +/** + * Resolves the effective pipeline for a run: which generator each generator + * reads its input from, and which generators the run finally collects. + * + * By default a generator reads from its `dependsOn`. A generator that declares + * one or more `dependent`s is spliced into each dependent's pipeline (see + * {@link splice}). Several generators may splice at the same point; each later + * one is inserted upstream of the earlier ones, forming a chain. + * + * A requested generator that declares dependents is delivered through them, + * so the run collects their output rather than the generator's own: through + * the dependents that are already part of the run when there are any, and + * through all of them otherwise. + * + * @param {string[]} targets - Requested targets, as resolved specifiers + * @param {Map} generators - Loaded generators, + * including every dependency and dependent of the targets + * @returns {{ targets: string[], inputOf: Map }} + * The generators to collect, and each generator's effective input generator + */ +export const resolvePipeline = (targets, generators) => { + /** @type {Map} */ + const inputOf = new Map(); + + for (const [specifier, { dependsOn }] of generators) { + inputOf.set( + specifier, + dependsOn ? resolveGeneratorSpecifier(dependsOn) : undefined + ); + } + + for (const [specifier, generator] of generators) { + for (const dependent of dependentsOf(generator)) { + splice(specifier, dependent, inputOf, generator.name); + } + } + + const requested = dependencyClosure(targets, generators); + const collected = []; + + for (const target of targets) { + for (const terminal of resolveTerminals(target, generators, requested)) { + if (!collected.includes(terminal)) { + collected.push(terminal); + } + } + } + + return { targets: collected, inputOf }; +}; diff --git a/packages/core/src/generators/types.d.ts b/packages/core/src/generators/types.d.ts index c1b155e99..0db82b9f6 100644 --- a/packages/core/src/generators/types.d.ts +++ b/packages/core/src/generators/types.d.ts @@ -97,6 +97,26 @@ declare global { */ dependsOn: string | undefined; + /** + * The generator(s) this generator's output is delivered through, declared + * as import specifiers. This is the inverse of `dependsOn`: rather than + * pulling a dependency in, it pushes this generator into another + * generator's pipeline. + * + * The orchestrator splices this generator in front of the first generator + * in each dependent's dependency chain that consumes the same `dependsOn` + * as this one. For example, a generator with + * `dependsOn: '@doc-kit/core/metadata'` and + * `dependent: '@doc-kit/generator-react/html'` is inserted between + * `metadata` and `jsx-ast` (the consumer of `metadata` on the way to + * `html`), so its output is what `jsx-ast` — and therefore `html` — sees. + * + * Requesting a generator that declares dependents runs them as well, and + * the run's result is their output: when some of the dependents are + * already part of the run, only those; otherwise all of them. + */ + dependent?: string | string[]; + /** * Generators are abstract and the different generators have different sort of inputs and outputs. * For example, a MDX generator would take the raw AST and output MDX with React Components; diff --git a/packages/react/README.md b/packages/react/README.md index f43a22ac9..bce35c24a 100644 --- a/packages/react/README.md +++ b/packages/react/README.md @@ -12,12 +12,13 @@ npm install --save-dev @doc-kit/core @doc-kit/generator-react ## Generators -| Target | Output | -| ---------- | -------------------------------------------------------------------------------------------- | -| `html` | The full documentation site — server-rendered pages hydrated with Preact, bundled with Vite. | -| `orama-db` | An [Orama](https://orama.com) search index, consumed by the `html` site's search box. | -| `llms-txt` | An [`llms.txt`](https://llmstxt.org/) index for Large Language Models. | -| `sitemap` | A `sitemap.xml` for search engines. | +| Target | Output | +| --------------- | -------------------------------------------------------------------------------------------- | +| `html` | The full documentation site — server-rendered pages hydrated with Preact, bundled with Vite. | +| `orama-db` | An [Orama](https://orama.com) search index, consumed by the `html` site's search box. | +| `llms-txt` | An [`llms.txt`](https://llmstxt.org/) index for Large Language Models. | +| `sitemap` | A `sitemap.xml` for search engines. | +| `section-pages` | The `html` site and sitemap, plus one page per section of every module. | The package also provides `jsx-ast`, the intermediate stage that turns parsed API metadata into JSX; it runs automatically when `html` needs it and is not a diff --git a/packages/react/package.json b/packages/react/package.json index 180d18a72..a19ba50c9 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -10,6 +10,7 @@ }, "license": "MIT", "exports": { + "./section-pages": "./src/section-pages/index.mjs", "./html": "./src/html/index.mjs", "./html/ui/*": "./src/html/ui/*", "./html/bundlers/vite": "./src/html/bundlers/vite.mjs", @@ -29,15 +30,16 @@ ], "dependencies": { "@11ty/is-land": "^5.0.1", + "@doc-kit/core": "workspace:*", "@fontsource-variable/open-sans": "^5.3.0", "@fontsource/ibm-plex-mono": "^5.3.0", "@heroicons/react": "^2.2.0", - "@doc-kit/core": "workspace:*", "@node-core/rehype-shiki": "^1.4.3", "@node-core/ui-components": "^1.7.6", "@orama/orama": "^3.1.18", "@orama/ui": "^1.5.4", "estree-util-to-js": "^2.0.0", + "github-slugger": "^2.0.0", "hast-util-to-string": "^3.0.1", "hastscript": "^9.0.1", "mdast-util-slice-markdown": "^2.0.1", diff --git a/packages/react/src/html/README.md b/packages/react/src/html/README.md index e89bca868..183324c41 100644 --- a/packages/react/src/html/README.md +++ b/packages/react/src/html/README.md @@ -175,20 +175,24 @@ omitted rather than rendered empty. ### Bundler adapters -- `getEntryId` {Function} Return the module identifier placed in the populated - HTML for an API name. +- `getEntryId` {Function} Return the module identifier placed in every + populated HTML page's client script tag. - `render` {Function} Bundle and execute the server `entries`, returning a `Map` of API name to rendered HTML. -- `build` {Function} Bundle the client `entries`, process the populated `pages`, +- `build` {Function} Bundle the client `entry`, process the populated `pages`, and write the complete output. The `bundler` option accepts a small Doc Kit adapter rather than configuration for a particular build system. -Both `render` and `build` receive `{ entries, virtualImports, config }`; `build` -also receives `pages`. Entry maps use `${api}.jsx` keys, rendered server results -use `api` keys, and page maps use output-relative HTML file names. `config` is -the resolved `html` configuration. +`render` receives `{ entries, virtualImports, config }`; `build` receives +`{ entry, virtualImports, pages, minifyPages, config }`. The server entry map +uses `${api}.jsx` keys and rendered server results use `api` keys. The client +`entry` is a single program shared by every page, served at the identifier +`getEntryId` returns. Page maps use output-relative HTML file names; +`minifyPages` takes such a map and returns its minified counterpart, spreading +the work across Doc Kit's worker pool — call it on the final HTML when +`config.minify` is set. `config` is the resolved `html` configuration. The adapter must compile the generated Preact JSX and CSS imports and resolve the supplied theme aliases and virtual modules. The generated `#theme/config` @@ -201,16 +205,17 @@ adding webpack to Doc Kit: ```js // webpack-bundler.mjs export const createWebpackBundler = webpackOptions => ({ - getEntryId: api => `virtual:doc-kit/client/${api}.jsx`, + getEntryId: () => 'virtual:doc-kit/client/index.jsx', async render({ entries, virtualImports, config }) { // Materialize or load the in-memory modules, run webpack's server target, // execute each emitted entry, and return Map. }, - async build({ entries, virtualImports, pages, config }) { + async build({ entry, virtualImports, pages, minifyPages, config }) { // Run webpack's browser target, inject its emitted assets into `pages`, - // and write the HTML and assets to config.output. + // minify them with `minifyPages` when `config.minify` is set, and write + // the HTML and assets to config.output. }, }); ``` @@ -385,6 +390,11 @@ import { project, repository, editURL } from '#theme/config'; - `documentationIndex` {Array} Entries rendered by the built-in `` component — every page with a stability index, each `{ api, name, index, description }`. +- `chunks` {Object} Per-module section pages produced by the + [`section-pages`](./section-pages.md) generator, keyed by the module's path: each + `{ label, items }`, where `items` is the module's section tree — `{ label, +path, items? }` entries nested by heading depth, in document order. Empty + unless `section-pages` ran. - `navigation` {Object} Mirrors the configured `navigation` (consumed by the built-in `SideBar` and `NavBar`). - `useAbsoluteURLs` {boolean} Whether internal links use absolute URLs (mirrors @@ -422,7 +432,9 @@ export default ({ metadata }) => ( ## Layout props - `metadata` {Object} Serialized page metadata — all YAML frontmatter properties - plus `addedIn`, `basename`, `path`, and any custom user-defined fields. + plus `addedIn`, `basename`, `path`, and any custom user-defined fields. Pages + produced by the [`section-pages`](./section-pages.md) generator also carry `chunk`, which + describes the module and section they were split from. - `headings` {Array} Pre-computed table of contents heading entries. - `readingTime` {string|undefined} Estimated reading time (e.g. `'5 min read'`). Only present when the `jsx-ast` generator's `showReadingTime` option is diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index 1f2856fa4..3ad6d1766 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -9,6 +9,7 @@ import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; +import { generate as chunk } from '../../section-pages/generate.mjs'; import { createViteBundler } from '../bundlers/vite.mjs'; import { generate } from '../generate.mjs'; @@ -21,12 +22,16 @@ const toCodeItem = content => ({ code: toJs(content, { handlers: jsx }).value, }); -const createEntry = (api, name) => { +const createEntry = ( + api, + text, + { depth = 1, slug = api, type, name = text } = {} +) => { const heading = { type: 'heading', - depth: 1, - children: [{ type: 'text', value: name }], - data: { name, text: name, slug: api }, + depth, + children: [{ type: 'text', value: text }], + data: { name, text, slug, type }, }; return { @@ -48,12 +53,12 @@ const createEntry = (api, name) => { }; }; -const createTestConfiguration = async context => { +const createTestConfiguration = async (context, target = ['html']) => { const output = await mkdtemp(join(tmpdir(), 'doc-kit-web-test-')); context.after(() => rm(output, { recursive: true, force: true })); const config = await setConfig({ - target: ['html'], + target, output, version: 'v22.0.0', changelog: [], @@ -91,6 +96,87 @@ describe('web generate', () => { assert.match(notFoundHTML, /src=\.\/assets\//); }); + it('renders chunk pages with navigation back to their module', async context => { + const { config, output } = await createTestConfiguration(context, [ + 'section-pages', + ]); + + // `fs.readFile()` is at depth 3 + config['section-pages'].maxDepth = 3; + + const entries = [ + createEntry('fs', 'File system'), + createEntry('fs', 'Callback API', { depth: 2, slug: 'callback-api' }), + createEntry('fs', '`fs.readFile()`', { + depth: 3, + slug: 'fsreadfile', + type: 'method', + name: 'readFile', + }), + createEntry('fs', '`fs.watch()`', { + depth: 2, + slug: 'fswatch', + type: 'method', + name: 'watch', + }), + ]; + + // A link from one section to another, and one to a sibling module + entries[2].content.children.push({ + type: 'paragraph', + children: [ + { + type: 'link', + url: '#fswatch', + children: [{ type: 'text', value: 'n' }], + }, + { + type: 'link', + url: 'net.html', + children: [{ type: 'text', value: 'x' }], + }, + ], + }); + + const pages = Map.groupBy(await chunk(entries), entry => entry.api); + const contents = await Promise.all( + [...pages.values()].map(group => buildContent(group, group[0])) + ); + + await generate(contents.map(toCodeItem)); + + const [fsHTML, readFileHTML] = await Promise.all([ + readFile(join(output, 'fs.html'), 'utf8'), + readFile(join(output, 'fs/readFile.html'), 'utf8'), + ]); + + // Assets and module files resolve from the nested directory + assert.match(readFileHTML, /src=\.\.\/assets\//); + assert.match(readFileHTML, /href=\.\.\/fs\.json/); + assert.match(readFileHTML, /href=\.\.\/fs\.html#fsreadfile/); + + // The sidebar nests the module's sections, repeating the module itself + assert.match(readFileHTML, /]*open/); + assert.match( + readFileHTML, + /href=\.\.\/fs\.html[^>]*>(<[^>]*>)*File system/ + ); + assert.match(readFileHTML, /href=callback-api\.html/); + + // Previous/next step through the module's sections + assert.match(readFileHTML, /Callback API/); + assert.match(readFileHTML, /href=watch\.html/); + assert.match(fsHTML, /Next/); + + // Links were re-targeted for the chunk page + assert.match(readFileHTML, /href=watch\.html#fswatch/); + assert.match(readFileHTML, /href=\.\.\/net\.html/); + + // The full page is untouched, and lists sections in its own sidebar + assert.match(fsHTML, /href=fs\.json/); + assert.match(fsHTML, /href=fs\/readFile\.html/); + }); + it('renders the configurable head without hardcoded defaults', async context => { const { config, output } = await createTestConfiguration(context); config.html.head = { @@ -159,9 +245,9 @@ describe('web generate', () => { const calls = []; config.html.bundler = { - getEntryId(api) { - calls.push(`entry:${api}`); - return `/custom/${api}.js`; + getEntryId() { + calls.push('entry'); + return '/custom/index.js'; }, async render({ entries, virtualImports, config: receivedConfig }) { @@ -179,10 +265,10 @@ describe('web generate', () => { ]); }, - async build({ entries, virtualImports, pages, config: receivedConfig }) { + async build({ entry, virtualImports, pages, config: receivedConfig }) { calls.push('client'); assert.strictEqual(receivedConfig, config.html); - assert.ok(entries.has('fs.jsx')); + assert.match(entry, /registerIslands\(/); assert.match( virtualImports['#theme/config'], /export const server = false;/ @@ -203,7 +289,7 @@ describe('web generate', () => { const html = await readFile(join(output, 'fs.html'), 'utf8'); assert.match(html, /data-custom-ssr/); - assert.match(html, /src="\/custom\/fs\.js"/); - assert.deepStrictEqual(calls, ['server', 'entry:fs', 'client']); + assert.match(html, /src="\/custom\/index\.js"/); + assert.deepStrictEqual(calls, ['server', 'entry', 'client']); }); }); diff --git a/packages/react/src/html/bundlers/vite.mjs b/packages/react/src/html/bundlers/vite.mjs index 592ffcff5..893dd66e7 100644 --- a/packages/react/src/html/bundlers/vite.mjs +++ b/packages/react/src/html/bundlers/vite.mjs @@ -3,7 +3,6 @@ import { tmpdir } from 'node:os'; import { basename, isAbsolute, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { minifyHTML } from '@doc-kit/core/utils/html-minifier.mjs'; import { build as viteBuild, defaultClientConditions, @@ -12,25 +11,23 @@ import { } from 'vite'; import { FONT_DIRECTORY } from '../constants.mjs'; +import { createPageMinifier } from '../utils/minify.mjs'; const VIRTUAL_PREFIX = 'virtual:doc-kit/'; const RESOLVED_VIRTUAL_PREFIX = '\0doc-kit:'; const PACKAGE_ANCHOR = fileURLToPath(import.meta.url); +// The single client entry every HTML page loads. One identifier means one +// entry chunk (plus its shared dependencies) for the whole site, rather than +// a copy per page. +const CLIENT_ENTRY_ID = `${VIRTUAL_PREFIX}client/index.jsx`; + /** * Resolves a package specifier */ const resolveFromPackage = specifier => fileURLToPath(import.meta.resolve(specifier)); -/** - * Returns the virtual Vite client entry imported by one HTML page. - * - * @param {string} api - * @returns {string} - */ -const getViteEntryId = api => `${VIRTUAL_PREFIX}client/${api}.jsx`; - /** * Resolves relative theme aliases against Vite's configured project root. * @@ -55,6 +52,11 @@ const resolveThemeAliases = (aliases, root) => * @returns {import('vite').Plugin} */ export const createVirtualModulesPlugin = sources => { + // Package imports are anchored to the same importer whichever virtual + // module they come from, so each specifier resolves the same way every + // time: resolve it once, not once per page that imports it. + const anchored = new Map(); + return { name: 'doc-kit:virtual-modules', enforce: 'pre', @@ -78,7 +80,14 @@ export const createVirtualModulesPlugin = sources => { !id.startsWith(VIRTUAL_PREFIX) && /^[@\w]/.test(id) ) { - return this.resolve(id, PACKAGE_ANCHOR, { skipSelf: true }); + if (!anchored.has(id)) { + anchored.set( + id, + this.resolve(id, PACKAGE_ANCHOR, { skipSelf: true }) + ); + } + + return anchored.get(id); } }, /** @@ -100,13 +109,15 @@ export const createVirtualModulesPlugin = sources => { /** * Finalizes Vite's generated HTML before its normal write phase. * + * @param {import('../types').ClientBundleOptions['minifyPages']} minifyPages * @returns {import('vite').Plugin} */ -const createHTMLFinalizerPlugin = () => ({ +const createHTMLFinalizerPlugin = minifyPages => ({ name: 'doc-kit:finalize-html', /** * Minifies every generated HTML entry after Vite has injected its scripts, - * stylesheets, and module preloads. + * stylesheets, and module preloads. The pages are handed over as one batch + * so the minifier can spread them across the worker pool. */ generateBundle: { order: 'post', @@ -115,44 +126,42 @@ const createHTMLFinalizerPlugin = () => ({ * @param {Record} bundle */ async handler(_, bundle) { - await Promise.all( - Object.values(bundle) - .filter( - item => item.type === 'asset' && item.fileName.endsWith('.html') - ) - .map(async asset => { - const source = - typeof asset.source === 'string' - ? asset.source - : Buffer.from(asset.source).toString('utf8'); - - asset.source = await minifyHTML(source); - }) + const assets = Object.values(bundle).filter( + item => item.type === 'asset' && item.fileName.endsWith('.html') ); + + const minified = await minifyPages( + new Map( + assets.map(asset => [ + asset.fileName, + typeof asset.source === 'string' + ? asset.source + : Buffer.from(asset.source).toString('utf8'), + ]) + ) + ); + + for (const asset of assets) { + asset.source = minified.get(asset.fileName); + } }, }, }); /** - * Converts generated page programs into named Vite inputs and virtual modules. + * Converts generated page programs into named Vite inputs and virtual modules + * for the SSR build: each page needs a distinct virtual JSX path of its own. * * @param {Map} codeMap - * @param {'client'|'server'} environment */ -const createBuildEntries = (codeMap, environment) => { +const createServerEntries = codeMap => { const input = {}; const sources = new Map(); for (const [fileName, code] of codeMap) { - const name = basename(fileName, '.jsx'); - // Client IDs must match entrypoints embedded in rendered HTML; server IDs - // only need a distinct virtual JSX path for the SSR build. - const id = - environment === 'client' - ? getViteEntryId(name) - : `${VIRTUAL_PREFIX}server/${fileName}`; - - input[name] = id; + const id = `${VIRTUAL_PREFIX}server/${fileName}`; + + input[basename(fileName, '.jsx')] = id; sources.set(id, code); } @@ -168,6 +177,7 @@ const createBuildEntries = (codeMap, environment) => { * @param {Record|Array} options.input * @param {boolean} options.server * @param {string} [options.serverOutDir] + * @param {import('../types').ClientBundleOptions['minifyPages']} [options.minifyPages] * @param {import('../types').ResolvedWebConfiguration} options.config * @param {import('vite').UserConfig} options.vite * @returns {import('vite').InlineConfig} @@ -177,6 +187,7 @@ export const createViteConfig = ({ input, server, serverOutDir, + minifyPages = createPageMinifier(), config: webConfig, vite = {}, }) => { @@ -202,7 +213,9 @@ export const createViteConfig = ({ plugins: [ createVirtualModulesPlugin(sources), ...(vite.plugins ?? []), - ...(!server && webConfig.minify ? [createHTMLFinalizerPlugin()] : []), + ...(!server && webConfig.minify + ? [createHTMLFinalizerPlugin(minifyPages)] + : []), ], resolve: mergeConfig( @@ -341,7 +354,7 @@ export const render = async ({ config, vite = {}, }) => { - const { input, sources } = createBuildEntries(entries, 'server'); + const { input, sources } = createServerEntries(entries); for (const [id, code] of Object.entries(virtualImports)) { sources.set(id, code); @@ -387,21 +400,23 @@ export const render = async ({ * hashed scripts, stylesheets, and module preloads, then writes the site. * * @param {object} options - * @param {Map} options.entries + * @param {string} options.entry * @param {Record} options.virtualImports * @param {Map} options.pages + * @param {import('../types').ClientBundleOptions['minifyPages']} [options.minifyPages] * @param {import('../types').ResolvedWebConfiguration} options.config * @param {import('vite').UserConfig} options.vite * @returns {Promise} */ export const build = async ({ - entries, + entry, virtualImports, pages, + minifyPages, config, vite = {}, }) => { - const { sources } = createBuildEntries(entries, 'client'); + const sources = new Map([[CLIENT_ENTRY_ID, entry]]); const root = resolve(vite.root ?? process.cwd()); const input = []; @@ -420,6 +435,7 @@ export const build = async ({ sources, input, server: false, + minifyPages, config, vite, }) @@ -433,7 +449,10 @@ export const build = async ({ * @returns {import('../types').WebBundler} */ export const createViteBundler = (options = {}) => ({ - getEntryId: getViteEntryId, + /** + * The client entry every page loads. + */ + getEntryId: () => CLIENT_ENTRY_ID, /** * Runs the Vite server build. * diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index 287bcb693..0aa1d2f63 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -5,6 +5,7 @@ import { readFile } from 'node:fs/promises'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { copyStaticAssets } from './utils/copying.mjs'; +import { createPageMinifier } from './utils/minify.mjs'; import { createCodeConverter, processBundles } from './utils/processing.mjs'; /** @@ -13,11 +14,13 @@ import { createCodeConverter, processBundles } from './utils/processing.mjs'; * Receives `jsx-ast`'s output as `{ data, code }` items — the JSX AST was * already serialized to `code` in the jsx-ast worker, so no AST is held here. * Bundling and rendering then run once over the accumulated code, since shared - * component chunks, CSS, and the sidebar need every entry together. + * component chunks, CSS, and the sidebar need every entry together. The + * worker pool only comes back into play for the final minification of the + * rendered pages. * * @type {import('./types').Generator['generate']} */ -export async function generate(input) { +export async function generate(input, worker) { const config = getConfig('html'); const template = await readFile(config.templatePath, 'utf-8'); @@ -34,17 +37,12 @@ export async function generate(input) { datas.push(item.data); } - // Sidebar lists only the real module pages. - const sidebarEntries = datas - .filter(data => data.synthetic !== true) - .map(data => ({ data })); - await processBundles({ serverCodeMap: converter.serverCodeMap, - clientCodeMap: converter.clientCodeMap, + clientProgram: converter.clientProgram, datas, - sidebarEntries, template, + minifyPages: createPageMinifier(worker), }); await copyStaticAssets(config); diff --git a/packages/react/src/html/index.mjs b/packages/react/src/html/index.mjs index 9f466c9e7..aad9dcae4 100644 --- a/packages/react/src/html/index.mjs +++ b/packages/react/src/html/index.mjs @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { generate } from './generate.mjs'; +import { processChunk } from './utils/minify.mjs'; /** * Web generator - transforms JSX AST entries into complete web bundles. @@ -86,4 +87,10 @@ export default { }), generate, + + // Minifying the rendered pages is the one step of the bundle that scales + // with the page count, so it is farmed out to the worker pool. + hasParallelProcessor: true, + + processChunk, }; diff --git a/packages/react/src/html/types.d.ts b/packages/react/src/html/types.d.ts index 9113c2c71..54a8052da 100644 --- a/packages/react/src/html/types.d.ts +++ b/packages/react/src/html/types.d.ts @@ -39,18 +39,22 @@ export type ServerBundleOptions = { }; export type ClientBundleOptions = { - // Client-side JSX programs keyed by `${api}.jsx`. - entries: Map; - // In-memory modules that the bundler must make available to the entries. + // The client-side program every page loads, hydrating its server-rendered + // markup; the bundler serves it at the identifier `getEntryId()` returns. + entry: string; + // In-memory modules that the bundler must make available to the entry. virtualImports: Record; // Populated HTML keyed by its output-relative file name. pages: Map; + // Minifies final pages (keyed like `pages`) across the worker pool. Bundlers + // call it on the HTML they are about to write when `config.minify` is set. + minifyPages: (pages: Map) => Promise>; config: ResolvedWebConfiguration; }; export type WebBundler = { - // Returns the module identifier embedded in one page's client script tag. - getEntryId(api: string): string; + // Returns the module identifier embedded in every page's client script tag. + getEntryId(): string; // Returns rendered HTML keyed by API name. render(options: ServerBundleOptions): Promise>; // Bundles the client entries and writes the complete site. diff --git a/packages/react/src/html/ui/components/Layout/index.jsx b/packages/react/src/html/ui/components/Layout/index.jsx index 2621d028c..20a0dac11 100644 --- a/packages/react/src/html/ui/components/Layout/index.jsx +++ b/packages/react/src/html/ui/components/Layout/index.jsx @@ -4,13 +4,58 @@ import Article from '@node-core/ui-components/Containers/Article'; import Banner from '../Banner'; import styles from './index.module.css'; +import { relativeOrAbsolute } from '../../utils/relativeOrAbsolute.mjs'; -import { navigation } from '#theme/config'; +import { navigation, chunks } from '#theme/config'; import Footer from '#theme/Footer'; import MetaBar from '#theme/Metabar'; import NavBar from '#theme/Navigation'; import SideBar from '#theme/Sidebar'; +/** + * Flattens a section tree into document order. + * + * @param {Array<{ label: string, path: string, items?: Array }>} sections + * @returns {Array<{ label: string, path: string }>} + */ +const flatten = sections => + sections.flatMap(({ items = [], ...section }) => [ + section, + ...flatten(items), + ]); + +/** + * Builds the ordered list of pages the previous/next links step through. + * + * A module that was split into chunks (and each of its chunk pages) steps + * through the module itself followed by its sections, in document order. + * Otherwise the configured sidebar order is used, when cross links are on. + * + * @param {import('../../types').SerializedMetadata} metadata + * @returns {Array<{ label: string, link: string, path: string }>} + */ +const getCrossLinkItems = metadata => { + const modulePath = metadata.chunk?.path ?? metadata.path; + const group = chunks[modulePath]; + + if (group) { + const toLink = path => `${relativeOrAbsolute(path, metadata.path)}.html`; + + return [ + { label: group.label, path: modulePath }, + ...flatten(group.items), + ].map(({ label, path }) => ({ label, link: toLink(path), path })); + } + + if (!navigation.showCrossLinks) { + return []; + } + + return (navigation.sidebar?.flatMap(({ items }) => items) ?? []).map( + item => ({ ...item, path: item.link }) + ); +}; + /** * Default page Layout component. * @@ -21,12 +66,10 @@ import SideBar from '#theme/Sidebar'; * @param {{ metadata: import('../../types').SerializedMetadata, headings: Array, readingTime?: string, children: import('preact').ComponentChildren }} props */ export default ({ metadata, headings, readingTime, children }) => { - const crossLinkItems = navigation.showCrossLinks - ? (navigation.sidebar?.flatMap(({ items }) => items) ?? []) - : []; + const crossLinkItems = getCrossLinkItems(metadata); const currentItem = crossLinkItems.findIndex( - ({ link }) => link === metadata.path + ({ path }) => path === metadata.path ); const [previousCrossLink, nextCrossLink] = [ diff --git a/packages/react/src/html/ui/components/MetaBar/index.jsx b/packages/react/src/html/ui/components/MetaBar/index.jsx index d9585d8f1..6b0849c5c 100644 --- a/packages/react/src/html/ui/components/MetaBar/index.jsx +++ b/packages/react/src/html/ui/components/MetaBar/index.jsx @@ -4,9 +4,10 @@ import MetaBar from '@node-core/ui-components/Containers/MetaBar'; import GitHubIcon from '@node-core/ui-components/Icons/Social/GitHub'; import styles from './index.module.css'; +import { relativeOrAbsolute } from '../../utils/relativeOrAbsolute.mjs'; import { STABILITY_KINDS, STABILITY_LABELS } from '../constants.mjs'; -import { editURL } from '#theme/config'; +import { editURL, chunks } from '#theme/config'; const iconMap = { JSON: CodeBracketIcon, @@ -47,11 +48,20 @@ const HeadingValue = ({ value, stability }) => { * @param {{ metadata: import('../../types').SerializedMetadata, headings: Array, readingTime?: string }} props */ export default ({ metadata, headings = [], readingTime }) => { - const editThisPage = editURL?.replace('{path}', metadata.path); + // A chunk page is a section of its module page: the source file, JSON and + // Markdown renderings, and edit link are all the module's. + const modulePath = metadata.chunk?.path ?? metadata.path; + + const editThisPage = editURL?.replace('{path}', modulePath); + + const toModuleFile = extension => + metadata.chunk + ? `${relativeOrAbsolute(modulePath, metadata.path)}${extension}` + : `${metadata.basename}${extension}`; const viewAs = [ - ['JSON', `${metadata.basename}.json`], - ['MD', `${metadata.basename}.md`], + ['JSON', toModuleFile('.json')], + ['MD', toModuleFile('.md')], ]; return ( @@ -66,6 +76,11 @@ export default ({ metadata, headings = [], readingTime }) => { items={{ 'Reading Time': readingTime, 'Added In': metadata.added ?? metadata.introduced_in, + 'Part Of': metadata.chunk && ( + + {chunks[modulePath]?.label ?? metadata.chunk.api} + + ), 'View As': !metadata.synthetic && (
    {viewAs.map(([viewTitle, path]) => { diff --git a/packages/react/src/html/ui/components/SideBar/index.jsx b/packages/react/src/html/ui/components/SideBar/index.jsx index 708291fba..e50fa9b04 100644 --- a/packages/react/src/html/ui/components/SideBar/index.jsx +++ b/packages/react/src/html/ui/components/SideBar/index.jsx @@ -6,7 +6,14 @@ import withIsland from '../../islands/withIsland.jsx'; import { relativeOrAbsolute } from '../../utils/relativeOrAbsolute.mjs'; import { renderLabel } from '../../utils/renderLabel.jsx'; -import { project, version, versions, navigation, pages } from '#theme/config'; +import { + project, + version, + versions, + navigation, + pages, + chunks, +} from '#theme/config'; /** * Extracts the major version number from a version string. @@ -32,16 +39,56 @@ const buildGroups = metadata => { ? `${metadata.basename}.html` : `${relativeOrAbsolute(path, metadata.path)}.html`; - const toItem = ({ label, link, items }) => ({ - label: renderLabel(label), - link: /^https?:/.test(link) ? link : toLink(link), - ...(items && { items: items.map(toItem) }), + // The module being viewed (directly, or through one of its chunk pages) + const currentModule = metadata.chunk?.path ?? metadata.path; + + /** + * Nests an item's children under it. An item with children renders as a + * disclosure rather than a link, so — as nodejs.org's site navigation does — + * the item repeats itself as the first child to keep its own page reachable. + * + * @param {{ label: string, link: string }} item + * @param {Array<{ label: string, link: string, items?: Array }>} children + */ + const nest = (item, children) => ({ + ...item, + items: [{ label: item.label, link: item.link }, ...children], }); + /** + * Converts a chunk section (and its sub-sections) into sidebar items. + * + * @param {{ label: string, path: string, items?: Array }} section + */ + const toSection = ({ label, path, items }) => { + const item = { label, link: path }; + + return items ? nest(item, items.map(toSection)) : item; + }; + + /** + * @param {{ label: string, link: string, items?: Array }} item + * @param {boolean} expand - Whether to nest the current module's sections + */ + const toItem = ({ label, link, items }, expand = true) => { + const entry = { label, link }; + + const { items: children } = + items || !expand || link !== currentModule || !chunks[link] + ? { items } + : nest(entry, chunks[link].items.map(toSection)); + + return { + label: renderLabel(label), + link: /^https?:/.test(link) ? link : toLink(link), + ...(children && { items: children.map(child => toItem(child, false)) }), + }; + }; + if (navigation.sidebar) { return navigation.sidebar.map(({ groupName, items }) => ({ groupName, - items: items.map(toItem), + items: items.map(item => toItem(item)), })); } diff --git a/packages/react/src/html/ui/types.d.ts b/packages/react/src/html/ui/types.d.ts index 2ff0138fa..526f8c2f2 100644 --- a/packages/react/src/html/ui/types.d.ts +++ b/packages/react/src/html/ui/types.d.ts @@ -28,11 +28,24 @@ declare module '#theme/config' { }>; export const editURL: string; export const pages: Array<[string, string]>; + export const chunks: Record; export const languageDisplayNameMap: Map; export const remoteConfigUrl: string; export const server: boolean; } +// A module's section pages, nested by heading depth (see `section-pages`) +export type ChunkSection = { + label: string; + path: string; + items?: Array; +}; + +export type ChunkGroup = { + label: string; + items: Array; +}; + // Omit Primitives from Metadata type Primitive = string | number | boolean | symbol | bigint | null | undefined; type OnlyPrimitives = { diff --git a/packages/react/src/html/utils/__tests__/config.test.mjs b/packages/react/src/html/utils/__tests__/config.test.mjs index 294a22933..999c9de57 100644 --- a/packages/react/src/html/utils/__tests__/config.test.mjs +++ b/packages/react/src/html/utils/__tests__/config.test.mjs @@ -15,39 +15,44 @@ mock.module('@node-core/rehype-shiki', { }); const { + default: createConfigSource, buildVersionEntries, buildPageList, + buildChunkGroups, buildDocumentationIndex, buildLanguageDisplayNameMap, } = await import('../config.mjs'); -await setConfig({ +const config = await setConfig({ version: 'v22.0.0', changelog: [ { version: new SemVer('20.0.0'), isLts: true, isCurrent: false }, { version: new SemVer('22.0.0'), isLts: false, isCurrent: true }, ], - generators: { - html: { - title: 'Node.js', - repository: 'nodejs/node', - ref: 'main', - baseURL: 'https://nodejs.org/docs', - editURL: 'https://github.com/nodejs/node/edit/main/doc/api{path}.md', - pageURL: '{baseURL}/latest-{version}/api{path}.html', - }, - }, }); +// Loading the real `html` generator would pull in the full rendering stack +// (which the `rehype-shiki` mock above cannot satisfy), so its resolved +// configuration is stubbed in directly. +config.html = { + ...config.global, + title: 'Node.js', + repository: 'nodejs/node', + ref: 'main', + baseURL: 'https://nodejs.org/docs', + editURL: 'https://github.com/nodejs/node/edit/main/doc/api{path}.md', + pageURL: '{baseURL}/latest-{version}/api{path}.html', + navigation: {}, +}; + /** - * Helper to create a minimal JSX content entry. + * Helper to create a minimal page metadata entry. */ -const makeEntry = (api, name, path) => ({ - data: { - api, - path, - heading: { depth: 1, data: { name } }, - }, +const makeEntry = (api, name, path, extra = {}) => ({ + api, + path, + heading: { depth: 1, data: { name } }, + ...extra, }); describe('buildVersionEntries', () => { @@ -112,11 +117,9 @@ describe('buildPageList', () => { const input = [ makeEntry('fs', 'File System', '/fs'), { - data: { - api: 'http', - path: '/http', - heading: { depth: 2, data: { name: 'HTTP' } }, - }, + api: 'http', + path: '/http', + heading: { depth: 2, data: { name: 'HTTP' } }, }, ]; @@ -131,40 +134,34 @@ describe('buildDocumentationIndex', () => { it('lists only pages with a stability index, with their descriptions', () => { const input = [ { - data: { - api: 'fs', - path: '/fs', - heading: { depth: 1, data: { name: 'File System' } }, - stability: { data: { index: '2' } }, - content: { - type: 'root', - children: [ - { - type: 'paragraph', - children: [{ type: 'text', value: 'File system APIs.' }], - }, - ], - }, + api: 'fs', + path: '/fs', + heading: { depth: 1, data: { name: 'File System' } }, + stability: { data: { index: '2' } }, + content: { + type: 'root', + children: [ + { + type: 'paragraph', + children: [{ type: 'text', value: 'File system APIs.' }], + }, + ], }, }, { - data: { - api: 'index', - path: '/index', - heading: { depth: 1, data: { name: 'Index' } }, - stability: null, - content: { type: 'root', children: [] }, - }, + api: 'index', + path: '/index', + heading: { depth: 1, data: { name: 'Index' } }, + stability: null, + content: { type: 'root', children: [] }, }, { - data: { - api: 'quic', - path: '/quic', - heading: { depth: 1, data: { name: 'QUIC' } }, - stability: { data: { index: '1.1' } }, - llm_description: 'QUIC protocol support.', - content: { type: 'root', children: [] }, - }, + api: 'quic', + path: '/quic', + heading: { depth: 1, data: { name: 'QUIC' } }, + stability: { data: { index: '1.1' } }, + llm_description: 'QUIC protocol support.', + content: { type: 'root', children: [] }, }, ]; @@ -189,15 +186,13 @@ describe('buildDocumentationIndex', () => { it('renders descriptions to HTML, without the links entries cannot nest', () => { const input = [ { - data: { - api: 'fs', - path: '/fs', - heading: { depth: 1, data: { name: 'File System' } }, - stability: { data: { index: '2' } }, - llm_description: - 'Enables interacting with the `file system`, see [fs](/fs).', - content: { type: 'root', children: [] }, - }, + api: 'fs', + path: '/fs', + heading: { depth: 1, data: { name: 'File System' } }, + stability: { data: { index: '2' } }, + llm_description: + 'Enables interacting with the `file system`, see [fs](/fs).', + content: { type: 'root', children: [] }, }, ]; @@ -210,6 +205,60 @@ describe('buildDocumentationIndex', () => { }); }); +describe('buildChunkGroups', () => { + const chunk = (name, index, depth = 2, label = name) => + makeEntry(`fs-${name}`, name, `/fs/${name}`, { + title: label, + chunk: { api: 'fs', path: '/fs', slug: name, index, depth }, + }); + + it('nests chunk pages under their module by depth, in document order', () => { + const groups = buildChunkGroups([ + chunk('readFile', 2, 3, 'fs.readFile'), + makeEntry('fs', 'File System', '/fs'), + chunk('notes', 3, 2, 'Notes'), + chunk('callback-api', 1, 2, 'Callback API'), + makeEntry('http', 'HTTP', '/http'), + ]); + + assert.deepStrictEqual(groups, { + '/fs': { + label: 'File System', + items: [ + { + label: 'Callback API', + path: '/fs/callback-api', + items: [{ label: 'fs.readFile', path: '/fs/readFile' }], + }, + { label: 'Notes', path: '/fs/notes' }, + ], + }, + }); + }); + + it('returns no groups when nothing was chunked', () => { + assert.deepStrictEqual( + buildChunkGroups([makeEntry('fs', 'File System', '/fs')]), + {} + ); + }); +}); + +describe('createConfigSource', () => { + it('lists only real module pages, and exposes the chunk groups', () => { + const source = createConfigSource([ + makeEntry('fs', 'File System', '/fs'), + makeEntry('all', 'All', '/all', { synthetic: true }), + makeEntry('fs-notes', 'Notes', '/fs/notes', { + chunk: { api: 'fs', path: '/fs', slug: 'notes', index: 0, depth: 2 }, + }), + ]); + + assert.match(source, /export const pages = \[\["File System","\/fs"\]\];/); + assert.match(source, /export const chunks = \{"\/fs":/); + }); +}); + describe('buildLanguageDisplayNameMap', () => { it('returns entries suitable for constructing a Map', () => { const result = buildLanguageDisplayNameMap(); diff --git a/packages/react/src/html/utils/config.mjs b/packages/react/src/html/utils/config.mjs index e8851d42e..7b3d9f114 100644 --- a/packages/react/src/html/utils/config.mjs +++ b/packages/react/src/html/utils/config.mjs @@ -37,23 +37,92 @@ export function buildVersionEntries(changelog, pageURLBase) { /** * Pre-compute sorted page list for sidebar navigation. * - * @param {Array} input + * @param {Array} input * @returns {Array<[string, string]>} */ export function buildPageList(input) { - const headNodes = getSortedHeadNodes(input.map(e => e.data)); + const headNodes = getSortedHeadNodes(input); return headNodes.map(node => [node.heading.data.name, node.path]); } +/** + * Nests a document-ordered list of sections by their heading depth. + * + * @template {{ depth: number }} T + * @param {Array} sections + * @returns {Array & { items?: Array }>} + */ +const toTree = sections => { + const root = { items: [] }; + const stack = [{ depth: 1, node: root }]; + + for (const { depth, ...section } of sections) { + while (stack.at(-1).depth >= depth) { + stack.pop(); + } + + (stack.at(-1).node.items ??= []).push(section); + stack.push({ depth, node: section }); + } + + return root.items; +}; + +/** + * Pre-compute the per-module section trees (see the `section-pages` generator): + * each module page's path maps to its label and its chunk pages, nested by + * heading depth and in document order, for the sidebar's section list and the + * previous/next links. + * + * @param {Array} input + * @returns {Record}>} + */ +export function buildChunkGroups(input) { + const labels = new Map( + input.map(entry => [entry.path, entry.heading.data.name]) + ); + + const groups = {}; + + for (const entry of input) { + if (!entry.chunk) { + continue; + } + + const group = (groups[entry.chunk.path] ??= { + label: labels.get(entry.chunk.path) ?? entry.chunk.api, + items: [], + }); + + group.items.push({ + label: entry.title ?? entry.heading.data.name, + path: entry.path, + depth: entry.chunk.depth, + index: entry.chunk.index, + }); + } + + // Pages arrive in render (completion) order, not document order + for (const group of Object.values(groups)) { + group.items = toTree( + group.items + .toSorted((a, b) => a.index - b.index) + .map(({ label, path, depth }) => ({ label, path, depth })) + ); + } + + return groups; +} + /** * Pre-compute the entries rendered by the `` component: * every page with a stability index, plus its description. * - * @param {Array} input + * @param {Array} input * @returns {Array<{api: string, name: string, index: string, description: string}>} */ export function buildDocumentationIndex(input) { - return getSortedHeadNodes(input.map(e => e.data)) + return getSortedHeadNodes(input) .filter(entry => entry.stability) .map(entry => ({ api: entry.api, @@ -91,13 +160,17 @@ export function buildLanguageDisplayNameMap() { * display labels and URL templates (with only `{path}` remaining for * per-page resolution by components). * - * @param {Array} input - JSX AST entries with .data metadata + * @param {Array} input - Per-page metadata * @param {boolean} [server=false] - Whether the module is for the server build. * @returns {string} JavaScript source code string with named exports */ export default function createConfigSource(input, server = false) { const { version: configVersion, ...config } = getConfig('html'); + // Only the real module pages are listed in the sidebar and the index; + // synthetic pages (`all`, `404`) and chunk pages are reachable elsewhere. + const pages = input.filter(entry => !entry.synthetic && !entry.chunk); + const editURL = config.editURL && populate(config.editURL, { @@ -128,8 +201,9 @@ export default function createConfigSource(input, server = false) { version: configVersion, versions: buildVersionEntries(config.changelog, pageURL), editURL, - pages: buildPageList(input), - documentationIndex: buildDocumentationIndex(input), + pages: buildPageList(pages), + documentationIndex: buildDocumentationIndex(pages), + chunks: buildChunkGroups(input), server, }; diff --git a/packages/react/src/html/utils/minify.mjs b/packages/react/src/html/utils/minify.mjs new file mode 100644 index 000000000..60d73f8c6 --- /dev/null +++ b/packages/react/src/html/utils/minify.mjs @@ -0,0 +1,49 @@ +'use strict'; + +import { minifyHTML } from '@doc-kit/core/utils/html-minifier.mjs'; + +/** + * Minifies a chunk of rendered pages. This is the `html` generator's worker + * entry point: minifying is the one CPU-bound step of the bundle that scales + * with the number of pages, and it parallelizes trivially — so it runs in the + * worker pool rather than serially on the main thread. + * + * @param {Array<[string, string]>} pages - `[fileName, html]` pairs + * @param {Array} indices - The pairs to process + * @returns {Promise>} The minified pairs + */ +export const processChunk = (pages, indices) => + Promise.all( + indices.map(async index => { + const [fileName, html] = pages[index]; + + return [fileName, await minifyHTML(html)]; + }) + ); + +/** + * Creates the page minifier handed to the bundler: it distributes the pages + * across the worker pool and collects the results as they complete. Without a + * pool — the generator was called directly rather than by the orchestrator — + * the pages are minified on the calling thread instead. + * + * @param {ParallelWorker} [worker] + * @returns {import('../types').ClientBundleOptions['minifyPages']} + */ +export const createPageMinifier = worker => async pages => { + const items = [...pages]; + + const chunks = worker + ? worker.stream(items) + : [await processChunk(items, [...items.keys()])]; + + const minified = new Map(); + + for await (const chunk of chunks) { + for (const [fileName, html] of chunk) { + minified.set(fileName, html); + } + } + + return minified; +}; diff --git a/packages/react/src/html/utils/processing.mjs b/packages/react/src/html/utils/processing.mjs index fd5798035..933e01317 100644 --- a/packages/react/src/html/utils/processing.mjs +++ b/packages/react/src/html/utils/processing.mjs @@ -11,14 +11,14 @@ import { THEME_SCRIPT } from '../ui/theme-script.mjs'; /** * Creates the virtual imports for one bundle target. * - * @param {Array<{ data: import('@doc-kit/core/generators/metadata/types').MetadataEntry }>} sidebarEntries + * @param {Array} datas - Per-page metadata * @param {Record} virtualImports * @param {boolean} server * @returns {Record} */ -const createVirtualImports = (sidebarEntries, virtualImports, server) => ({ +const createVirtualImports = (datas, virtualImports, server) => ({ ...virtualImports, - '#theme/config': createConfigSource(sidebarEntries, server), + '#theme/config': createConfigSource(datas, server), }); /** @@ -52,6 +52,21 @@ export const resolvePageRoot = data => { return unresolvedRoot.endsWith('/') ? unresolvedRoot : `${unresolvedRoot}/`; }; +/** + * Escapes text for interpolation into HTML, as element content or as a quoted + * attribute value: a heading such as `What does it mean to "contextify" an + * object?` must not terminate the `` it is rendered into. + * + * @param {string} text + * @returns {string} + */ +export const escapeHTML = text => + text + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"'); + /** * Renders a self-closing HTML tag from an attribute bag. * @@ -106,31 +121,27 @@ export const buildHead = ({ meta = [], links = [], html = [] }) => * string upstream (in the `jsx-ast` worker), so the heavy AST never reaches * the main thread — only the code string and page metadata stream in here. * - * @returns {{ add: (item: { data: import('@doc-kit/core/generators/metadata/types').MetadataEntry, code: string }) => void, serverCodeMap: Map, clientCodeMap: Map }} + * @returns {{ add: (item: { data: import('@doc-kit/core/generators/metadata/types').MetadataEntry, code: string }) => void, serverCodeMap: Map, clientProgram: string }} */ export function createCodeConverter() { const { buildServerProgram, clientProgram } = createProgramBuilder(); const serverCodeMap = new Map(); - const clientCodeMap = new Map(); return { /** - * Records the server/client programs for a single page's JSX code. + * Records the server program for a single page's JSX code. * * @param {{ data: import('@doc-kit/core/generators/metadata/types').MetadataEntry, code: string }} item */ add: ({ data, code }) => { - const fileName = `${data.api}.jsx`; - // Prepare code for server-side execution (wrapped for SSR) - serverCodeMap.set(fileName, buildServerProgram(code)); - - // Every page's entry is the same module; the bundler emits one chunk. - clientCodeMap.set(fileName, clientProgram); + serverCodeMap.set(`${data.api}.jsx`, buildServerProgram(code)); }, serverCodeMap, - clientCodeMap, + // The client entry is the same module for every page: the pages differ + // only in their server-rendered markup, which the entry hydrates. + clientProgram, }; } @@ -142,28 +153,24 @@ export function createCodeConverter() { * * @param {object} params * @param {Map} params.serverCodeMap - Server-side code per page. - * @param {Map} params.clientCodeMap - Client-side code per page. + * @param {string} params.clientProgram - The client entry shared by every page. * @param {Array} params.datas - Per-page metadata, in render order. - * @param {Array<{ data: import('@doc-kit/core/generators/metadata/types').MetadataEntry }>} params.sidebarEntries - Entries used to build the sidebar page list (real module pages only). * @param {string} params.template - The HTML template string for the output pages. + * @param {import('../types').ClientBundleOptions['minifyPages']} params.minifyPages - Minifies the final pages, off the main thread. */ export async function processBundles({ serverCodeMap, - clientCodeMap, + clientProgram, datas, - sidebarEntries, template, + minifyPages, }) { const config = getConfig('html'); const bundler = await resolveBundler(config.bundler); const serverPages = await bundler.render({ entries: serverCodeMap, - virtualImports: createVirtualImports( - sidebarEntries, - config.virtualImports, - true - ), + virtualImports: createVirtualImports(datas, config.virtualImports, true), config, }); @@ -177,8 +184,10 @@ export async function processBundles({ // template authors avoid nested template-literal escaping. const head = buildHead(config.head); - // Render the templates with the client identifiers supplied by the adapter. + // Render the templates with the client identifier supplied by the adapter. // The adapter then owns scripts, stylesheets, preloads, and imported assets. + const entrypoint = bundler.getEntryId(); + const pages = new Map( datas.map(data => { const root = resolvePageRoot(data); @@ -188,13 +197,15 @@ export async function processBundles({ return [ fileName, populateWithEvaluation(template, { - title: title - ? titleSuffix - ? `${title} | ${titleSuffix}` - : title - : titleSuffix, + title: escapeHTML( + title + ? titleSuffix + ? `${title} | ${titleSuffix}` + : title + : titleSuffix + ), dehydrated: serverPages.get(data.api) ?? '', - entrypoint: bundler.getEntryId(data.api), + entrypoint, speculationRules: SPECULATION_RULES, themeScript: THEME_SCRIPT, preloads: buildPreloads(root), @@ -208,13 +219,10 @@ export async function processBundles({ ); await bundler.build({ - entries: clientCodeMap, - virtualImports: createVirtualImports( - sidebarEntries, - config.virtualImports, - false - ), + entry: clientProgram, + virtualImports: createVirtualImports(datas, config.virtualImports, false), pages, + minifyPages, config, }); } diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index 25d0c2b99..bd073c980 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -59,8 +59,11 @@ export async function processChunk(slicedInput, itemIndices) { */ export async function* generate(input, worker) { // The `index` page is only generated when an `index` document is part of - // the input; the module list for the synthetic pages excludes it. - const moduleInput = input.filter(entry => entry.api !== 'index'); + // the input; the module list for the synthetic pages excludes it, as well + // as chunk pages, whose content the full module pages already carry. + const moduleInput = input.filter( + entry => entry.api !== 'index' && !entry.chunk + ); // Create sliced input: each item contains head + its module's entries // This avoids sending all 4700+ entries to every worker diff --git a/packages/react/src/jsx-ast/utils/buildBarProps.mjs b/packages/react/src/jsx-ast/utils/buildBarProps.mjs index 8062a41fc..54d08f159 100644 --- a/packages/react/src/jsx-ast/utils/buildBarProps.mjs +++ b/packages/react/src/jsx-ast/utils/buildBarProps.mjs @@ -43,7 +43,7 @@ const shouldIncludeEntryInToC = ({ heading }) => * * @param {import('@doc-kit/core/generators/metadata/types').HeadingData} data */ -const headingLabel = data => { +export const headingLabel = data => { if (FUNCTION_HEADING_TYPES.has(data.type)) { const name = getFullName(data, data.name); diff --git a/packages/react/src/jsx-ast/utils/buildContent.mjs b/packages/react/src/jsx-ast/utils/buildContent.mjs index 5b29a2f9c..17eb44f16 100644 --- a/packages/react/src/jsx-ast/utils/buildContent.mjs +++ b/packages/react/src/jsx-ast/utils/buildContent.mjs @@ -233,7 +233,10 @@ export const transformHeadingNode = async (entry, node, index, parent) => { createChangeElement(entry) ); - if (entry.api === 'deprecations' && node.depth === 3) { + // Chunk pages promote their headings, so match on the original depth + const depth = node.depth + (entry.chunk?.depth ?? 1) - 1; + + if ((entry.chunk?.api ?? entry.api) === 'deprecations' && depth === 3) { // On the 'deprecations.md' page, "Type: " turns into an AlertBox // Extract the nodes representing the type text const { node } = slice( diff --git a/packages/react/src/section-pages/README.md b/packages/react/src/section-pages/README.md new file mode 100644 index 000000000..9ac338344 --- /dev/null +++ b/packages/react/src/section-pages/README.md @@ -0,0 +1,143 @@ +# `section-pages` Generator + +The `section-pages` generator splits every documented module into per-section pages. +It leaves the full module page untouched and adds one extra page for each +section next to it: + +```text +out/ +├── fs.html the complete module page, exactly as before +└── fs/ + ├── promises-api.html ## Promises API + ├── FileHandle.html ### Class: FileHandle, with all of its members + ├── fsPromises.readFile.html + ├── callback-api.html ## Callback API + ├── readFile.html ### fs.readFile(path[, options], callback) + └── ... +``` + +Each chunk page renders like any other page: same layout, sidebar, table of +contents, and search. On top of that: + +- the sidebar nests the current module's sections under its entry (the module + itself is repeated as the first item, so the full page stays one click away); +- previous/next links step through the module and its sections in order; +- the meta bar links a chunk back to its place on the full page ("Part Of"), + and its "View As" and "Edit this page" links point at the module's files; +- links inside a section keep working: fragment links follow the section they + point at to whichever page it landed on, and relative links are re-based. + +```bash +npx @doc-kit/cli generate -t section-pages -i "doc/api/*.md" -o out +``` + +## How it works + +`section-pages` does not render anything itself. It consumes the `metadata` +generator's entries, and for each module appends copies of the entries of every +section, re-homed under the section's own `api` and `path` (`fs/readFile`). +It then declares the generators it is delivered through as its +[`dependent`](../../../../docs/creating-generators.md#declaring-a-dependent)s: + +- `html`, where it is spliced between `metadata` and `jsx-ast`, so the web + generators render the chunk pages as if they had always been there; +- `sitemap`, where it is spliced in front of the generator itself, so the + chunk pages are listed. + +```text +ast → metadata → section-pages → jsx-ast → html + └─────────→ sitemap +``` + +Requesting `-t section-pages` on its own therefore produces both. Requested together +with either of them (`-t html -t section-pages`), it only feeds the ones that run. + +Other generators that read `metadata` — `orama-db`, `llms-txt`, and the JSON +outputs — are deliberately left alone: search results and `llms.txt` keep +pointing at the full pages, which carry the same anchors, and nothing is +indexed twice. + +## What becomes a chunk + +Within a module, a new chunk starts at: + +- every depth-2 heading (`## Callback API`, `## Notes`), and +- every heading up to `maxDepth` that documents an API entry — a method, class, + constructor, event, property, static method, or global (`### fs.readFile()`). + +Everything else stays in the chunk of the closest heading above it: prose +sub-sections (`#### File descriptors`), and — regardless of depth — the members +of a class, so that a class page lists its whole API (`### Class: fs.Dir` keeps +`#### dir.close()`). + +A section only becomes a chunk when it documents an API entry — its own +heading, or one nested anywhere below it, is a method, class, constructor, +event, property, or static method. Pure prose (`## Introduction`, `## Notes`, +the whole of "About this documentation") reads best in the context of the full +page, and stays there. A prose-headed section whose sub-sections are API +entries (`## Callback API`) is still a chunk, so that the sidebar can nest them +under it. + +The module's own introduction (everything before the first depth-2 heading) is +only on the full page. Modules that would yield fewer than two chunks — which +includes every module that documents no API at all — and modules listed in +`exclude`, are left alone. + +## File names + +A chunk's file name comes from its heading. API names keep their case, with the +module's own prefix dropped, so `fs.readFile()` becomes `fs/readFile.html` and +`Class: fs.Dir` becomes `fs/Dir.html`; names in another namespace keep it +(`fs/fsPromises.readFile.html`). Prose headings are slugged +(`fs/callback-api.html`). Duplicates within a module are suffixed +(`close.html`, `close-2.html`), comparing case-insensitively so the output is +safe on case-insensitive file systems. + +Heading anchors are unchanged, so `fs/readFile.html#fsreadfilepath-options-callback` +and `fs.html#fsreadfilepath-options-callback` point at the same section. + +## Configuring + +- `maxDepth` {number} The deepest heading level that may start a chunk. + Headings below it always stay with the section above them. **Default:** `2`, + which only splits at `##` headings; the tree above is what `3` produces. +- `exclude` {Array} `api` names of modules that are never split, on top of the + modules that document no API entry. **Default:** `[]`. + +```js +// doc-kit.config.mjs +export default { + target: ['section-pages'], + 'section-pages': { + // Also split at `###` headings that document an API entry + maxDepth: 3, + exclude: ['deprecations'], + }, +}; +``` + +## Chunk metadata + +Every entry of a chunk page carries a `chunk` property describing its origin, +which the `html` UI uses for its navigation: + +- `api` {string} The `api` of the module the chunk was split from (`fs`). +- `path` {string} The `path` of the module (`/fs`). +- `slug` {string} The section's anchor on the full page. +- `index` {number} The chunk's position within the module, in document order. +- `depth` {number} The original depth of the chunk's heading. + +The chunk's first entry also gets a `title` — its sidebar label and page +title (`fs.readFile`). The `html` generator exposes every module's section +tree through the `chunks` export of +[`#theme/config`](./html.md#themeconfig-virtual-module), for custom themes. + +## Trade-offs + +- Chunk pages duplicate content that is also on the full page, so the site + roughly doubles in page count and the build grows accordingly. +- No redirects are generated from `fs.html#anchor` to a chunk page: the full + page still exists, so existing links keep working, and both pages share + their anchors. +- Chunk pages do not get `.json` or `.md` renderings of their own; their + "View As" links lead to the module's. diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs new file mode 100644 index 000000000..5f5409227 --- /dev/null +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -0,0 +1,218 @@ +import assert from 'node:assert/strict'; +import { before, describe, it } from 'node:test'; + +import getConfig, { + setConfig, +} from '@doc-kit/core/utils/configuration/index.mjs'; + +import { generate } from '../generate.mjs'; + +/** + * Builds a metadata-shaped entry whose heading node is shared between + * `heading` and `content`, exactly like the metadata generator produces. + */ +const createEntry = (api, depth, text, { name = text, type, slug } = {}) => { + const heading = { + type: 'heading', + depth, + children: [{ type: 'text', value: text }], + data: { text, name, slug: slug ?? text.toLowerCase(), type }, + }; + + return { + api, + path: `/${api}`, + basename: api, + heading, + stability: null, + content: { + type: 'root', + children: [ + heading, + { + type: 'paragraph', + children: [{ type: 'text', value: `${text} body` }], + }, + ], + }, + }; +}; + +const createModule = () => [ + { + ...createEntry('fs', 1, 'File system', { slug: 'file-system' }), + introduced_in: 'v0.10.0', + }, + createEntry('fs', 2, 'Callback API'), + createEntry('fs', 3, '`fs.readFile(path, callback)`', { + name: 'readFile', + type: 'method', + }), + createEntry('fs', 4, 'File descriptors', { slug: 'file-descriptors' }), + createEntry('fs', 2, 'Class: `fs.Dir`', { + name: 'fs.Dir', + type: 'class', + slug: 'class-fsdir', + }), + createEntry('fs', 3, "Event: `'close'`", { name: 'close', type: 'event' }), +]; + +const link = url => ({ + type: 'paragraph', + children: [{ type: 'link', url, children: [{ type: 'text', value: url }] }], +}); + +describe('section-pages generate', () => { + before(async () => { + await setConfig({ target: ['section-pages'] }); + + // The module below splits at depth 3 + getConfig('section-pages').maxDepth = 3; + }); + + it('keeps every original entry and appends the chunk pages', async () => { + const input = createModule(); + const output = await generate(input); + + assert.deepEqual(output.slice(0, input.length), input); + + const chunkHeads = output.filter(e => e.chunk && e.heading.depth === 1); + + assert.deepEqual( + chunkHeads.map(e => e.path), + ['/fs/callback-api', '/fs/readFile', '/fs/Dir'] + ); + assert.deepEqual( + chunkHeads.map(e => e.api), + ['fs-callback-api', 'fs-readFile', 'fs-Dir'] + ); + assert.deepEqual( + chunkHeads.map(e => e.basename), + ['callback-api', 'readFile', 'Dir'] + ); + assert.deepEqual( + chunkHeads.map(e => e.title), + ['Callback API', 'fs.readFile', 'fs.Dir'] + ); + }); + + it('describes where each chunk came from', async () => { + const output = await generate(createModule()); + const readFile = output.find(e => e.path === '/fs/readFile'); + + assert.deepEqual(readFile.chunk, { + api: 'fs', + path: '/fs', + slug: '`fs.readfile(path, callback)`', + index: 1, + depth: 3, + }); + assert.equal(readFile.introduced_in, 'v0.10.0'); + }); + + it('promotes headings so the chunk reads as its own page', async () => { + const output = await generate(createModule()); + + const dir = output.filter(e => e.api === 'fs-Dir'); + assert.deepEqual( + dir.map(e => e.heading.depth), + [1, 2] + ); + assert.deepEqual( + dir.map(e => e.content.children[0].depth), + [1, 2] + ); + + const readFile = output.filter(e => e.api === 'fs-readFile'); + assert.deepEqual( + readFile.map(e => e.heading.depth), + [1, 2] + ); + }); + + it('does not mutate the original entries', async () => { + const input = createModule(); + const snapshot = structuredClone(input); + + await generate(input); + + assert.deepEqual(input, snapshot); + }); + + it('leaves modules with a single section alone', async () => { + const output = await generate([ + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]); + + assert.equal(output.length, 2); + }); + + it('leaves modules that document no API entry alone', async () => { + const output = await generate([ + createEntry('guide', 1, 'About this documentation'), + createEntry('guide', 2, 'Contributing'), + createEntry('guide', 2, 'Stability index'), + createEntry('guide', 3, 'Stability overview'), + ]); + + assert.equal(output.length, 4); + }); + + it('re-targets links authored for the module page', async () => { + const input = createModule(); + const readFile = input[2]; + + readFile.content.children.push( + link('#file-descriptors'), // same chunk + link('#class-fsdir'), // another chunk + link('#file-system'), // only on the full page + link('net.html#foo'), // relative to the module's directory + link('/absolute.html'), + link('https://example.com') + ); + + const output = await generate(input); + const chunk = output.find(e => e.path === '/fs/readFile'); + + assert.deepEqual( + chunk.content.children.slice(2).map(p => p.children[0].url), + [ + '#file-descriptors', + 'Dir.html#class-fsdir', + '../fs.html#file-system', + '../net.html#foo', + '/absolute.html', + 'https://example.com', + ] + ); + + // The full page keeps its own links + assert.equal(readFile.content.children[3].children[0].url, '#class-fsdir'); + }); + + it('never splits excluded modules', async () => { + getConfig('section-pages').exclude = ['fs']; + + try { + assert.equal((await generate(createModule())).length, 6); + } finally { + getConfig('section-pages').exclude = []; + } + }); + + it('honours maxDepth', async () => { + getConfig('section-pages').maxDepth = 2; + + try { + const output = await generate(createModule()); + + assert.deepEqual( + output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path), + ['/fs/callback-api', '/fs/Dir'] + ); + } finally { + getConfig('section-pages').maxDepth = 3; + } + }); +}); diff --git a/packages/react/src/section-pages/__tests__/naming.test.mjs b/packages/react/src/section-pages/__tests__/naming.test.mjs new file mode 100644 index 000000000..0e311c622 --- /dev/null +++ b/packages/react/src/section-pages/__tests__/naming.test.mjs @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { createUniqueNamer, toFileName } from '../utils/naming.mjs'; + +describe('toFileName', () => { + const typed = { type: 'method' }; + const prose = {}; + + it('keeps the case of API names and drops the module prefix', () => { + assert.equal(toFileName('fs.readFile', typed, 'fs'), 'readFile'); + assert.equal(toFileName('fs.Dir', { type: 'class' }, 'fs'), 'Dir'); + }); + + it('keeps other namespaces as part of the name', () => { + assert.equal( + toFileName('fsPromises.readFile', typed, 'fs'), + 'fsPromises.readFile' + ); + }); + + it('strips quotes and constructor keywords', () => { + assert.equal(toFileName("'close'", { type: 'event' }, 'net'), 'close'); + assert.equal( + toFileName('new v8.Deserializer', { type: 'ctor' }, 'v8'), + 'Deserializer' + ); + }); + + it('slugs prose headings, even when they would be safe file names', () => { + assert.equal(toFileName('Callback API', prose, 'fs'), 'callback-api'); + assert.equal(toFileName('Notes', prose, 'fs'), 'notes'); + }); + + it('slugs API names that are not safe file names', () => { + assert.equal( + toFileName('napi_create_string_utf8()', typed, 'n-api'), + 'napi_create_string_utf8' + ); + }); +}); + +describe('createUniqueNamer', () => { + it('suffixes repeated names, ignoring case', () => { + const unique = createUniqueNamer(); + + assert.equal(unique('close'), 'close'); + assert.equal(unique('close'), 'close-2'); + assert.equal(unique('Close'), 'Close-3'); + assert.equal(unique('open'), 'open'); + }); +}); diff --git a/packages/react/src/section-pages/__tests__/split.test.mjs b/packages/react/src/section-pages/__tests__/split.test.mjs new file mode 100644 index 000000000..28d0156e7 --- /dev/null +++ b/packages/react/src/section-pages/__tests__/split.test.mjs @@ -0,0 +1,160 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { splitIntoChunks } from '../utils/split.mjs'; + +const entry = (depth, text, type) => ({ + heading: { depth, data: { text, name: text, slug: text, type } }, +}); + +const names = chunk => chunk.entries.map(e => e.heading.data.text); + +describe('splitIntoChunks', () => { + it('starts a chunk at every depth-2 heading and drops the module intro', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'a()', 'method'), + entry(2, 'b()', 'method'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [['a()'], ['b()']]); + assert.deepEqual( + chunks.map(c => c.depth), + [2, 2] + ); + }); + + it('starts a chunk at API-typed headings, but keeps prose sub-sections', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'Callback API'), + entry(3, 'fs.readFile()', 'method'), + entry(4, 'File descriptors'), + entry(3, 'Caveats'), + entry(3, "Event: 'close'", 'event'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [ + ['Callback API'], + ['fs.readFile()', 'File descriptors', 'Caveats'], + ["Event: 'close'"], + ]); + }); + + it('never starts a chunk beyond maxDepth', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'Section'), + entry(3, 'a()', 'method'), + entry(4, 'b()', 'method'), + ], + 2 + ); + + assert.deepEqual(chunks.map(names), [['Section', 'a()', 'b()']]); + }); + + it('keeps a class together with its members', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'Class: net.Server', 'class'), + entry(3, 'new net.Server()', 'ctor'), + entry(3, "Event: 'close'", 'event'), + entry(3, 'server.listen()', 'method'), + entry(2, 'net.connect()', 'method'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [ + [ + 'Class: net.Server', + 'new net.Server()', + "Event: 'close'", + 'server.listen()', + ], + ['net.connect()'], + ]); + }); + + it('closes a class chunk at the next heading of the same depth or above', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'Promises API'), + entry(3, 'Class: FileHandle', 'class'), + entry(4, 'filehandle.read()', 'method'), + entry(3, 'fsPromises.access()', 'method'), + entry(2, 'Callback API'), + entry(3, 'fs.access()', 'method'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [ + ['Promises API'], + ['Class: FileHandle', 'filehandle.read()'], + ['fsPromises.access()'], + ['Callback API'], + ['fs.access()'], + ]); + }); + + it('leaves content before the first chunk to the full page', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(4, 'Stray'), + entry(2, 'Section'), + entry(3, 'a()', 'method'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [['Section'], ['a()']]); + }); + + it('leaves sections that document no API entry to the full page', () => { + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'Introduction'), + entry(3, 'Terminology'), + entry(2, 'Class: AsyncLocalStorage', 'class'), + entry(3, 'new AsyncLocalStorage()', 'ctor'), + entry(2, 'Notes'), + entry(3, 'Threadpool usage'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [ + ['Class: AsyncLocalStorage', 'new AsyncLocalStorage()'], + ]); + }); + + it('keeps a prose-headed section whose sub-sections document API entries', () => { + // Even when those sub-sections get chunks of their own: the section page + // then carries the introduction, and the sidebar nests the rest under it. + const chunks = splitIntoChunks( + [ + entry(1, 'Module'), + entry(2, 'Callback API'), + entry(3, 'fs.readFile()', 'method'), + entry(2, 'Exit codes'), + entry(3, '1 Uncaught Fatal Exception'), + ], + 3 + ); + + assert.deepEqual(chunks.map(names), [['Callback API'], ['fs.readFile()']]); + }); +}); diff --git a/packages/react/src/section-pages/constants.mjs b/packages/react/src/section-pages/constants.mjs new file mode 100644 index 000000000..fa2329381 --- /dev/null +++ b/packages/react/src/section-pages/constants.mjs @@ -0,0 +1,21 @@ +'use strict'; + +// Sections headed by one of these API types are kept together as a single +// chunk, regardless of `maxDepth`: a class page should list its members. +export const SELF_CONTAINED_TYPES = new Set(['class']); + +// Headings of one of these API types start a chunk of their own wherever they +// appear (up to `maxDepth`). Untyped headings only do so at depth 2, so prose +// sub-sections stay with the section they belong to. +export const CHUNKABLE_TYPES = new Set(['class', 'event', 'global', 'method']); + +// A chunk's file name is derived from its heading. Anything else is slugged. +export const SAFE_FILE_NAME = /^[\w.$-]+$/; + +// A URL that is neither absolute, site-absolute, nor a fragment: it was +// authored relative to the module page and must be re-based for a chunk page. +export const RELATIVE_URL = /^(?![a-z][a-z0-9+.-]*:|\/|#|\?)/i; + +// Modules that split into fewer chunks than this are left alone: a single +// chunk would just duplicate the full page. +export const MIN_CHUNKS = 2; diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs new file mode 100644 index 000000000..8c742feb0 --- /dev/null +++ b/packages/react/src/section-pages/generate.mjs @@ -0,0 +1,139 @@ +'use strict'; + +import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; +import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { visit } from 'unist-util-visit'; + +import { MIN_CHUNKS } from './constants.mjs'; +import { buildAnchorMap, rewriteUrl } from './utils/links.mjs'; +import { createUniqueNamer, toFileName } from './utils/naming.mjs'; +import { splitIntoChunks } from './utils/split.mjs'; +import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs'; + +// The nodes that carry a URL authored relative to the module page +const URL_NODE_TYPES = new Set(['link', 'image', 'definition']); + +/** + * Re-homes a cloned entry on its chunk page, in a single pass over its + * content: headings are promoted so the chunk's own heading renders as the + * page title (depth 1) with its sub-sections keeping their relative nesting, + * and URLs are re-targeted from the module page to the chunk page. + * + * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} entry - A cloned entry (mutated) + * @param {number} shift - How many levels to promote headings by + * @param {Parameters[1]} urls - See {@link rewriteUrl} + */ +const rehome = (entry, shift, urls) => { + const promoted = new Set(); + + /** + * Shifts one heading node, at most once. + * + * @param {import('mdast').Heading} node + */ + const promote = node => { + if (!promoted.has(node)) { + promoted.add(node); + node.depth = Math.max(1, node.depth - shift); + } + }; + + promote(entry.heading); + + visit(entry.content, node => { + if (node.type === 'heading') { + // The heading also lives in the content tree — usually as the very same + // node, which must not be shifted twice, but that is not guaranteed. + promote(node); + } else if (URL_NODE_TYPES.has(node.type) && node.url) { + node.url = rewriteUrl(node.url, urls); + } + }); +}; + +/** + * Builds the metadata entries of one chunk page. The originals are left + * untouched — the full page still needs them — so each entry is cloned and + * re-homed under the chunk's own `api` and `path`. + * + * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} module - The module's depth-1 entry + * @param {import('./types').Chunk} chunk + * @param {number} index - The chunk's position within the module + * @param {Map} anchors - Where each of the module's anchors lives + * @returns {Array} + */ +const buildChunkEntries = (module, chunk, index, anchors) => { + const { name, path } = chunk; + const api = `${module.api}-${name}`; + const shift = chunk.depth - 1; + const urls = { modulePath: module.path, chunkPath: path, anchors }; + + /** @type {import('@doc-kit/core/generators/metadata/types').ChunkInfo} */ + const info = { + api: module.api, + path: module.path, + slug: chunk.head.heading.data.slug, + index, + depth: chunk.depth, + }; + + return chunk.entries.map((original, position) => { + const entry = structuredClone(original); + + rehome(entry, shift, urls); + + Object.assign(entry, { api, path, basename: name, chunk: info }); + + if (position === 0) { + // The page title and sidebar label + entry.title = headingLabel(original.heading.data); + // Version pickers and "Added in" fall back to the module's own version + entry.introduced_in ??= module.introduced_in; + } + + return entry; + }); +}; + +/** + * Adds per-section chunk pages for every module, next to the full pages. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const { maxDepth, exclude } = getConfig('section-pages'); + + const output = [...input]; + + for (const entries of groupNodesByModule(input).values()) { + const module = entries.find(entry => entry.heading.depth === 1); + + if (!module || module.synthetic || exclude.includes(module.api)) { + continue; + } + + const chunks = splitIntoChunks(entries, maxDepth); + + if (chunks.length < MIN_CHUNKS) { + continue; + } + + // Name every chunk first: links between them need all the paths + const unique = createUniqueNamer(); + + for (const chunk of chunks) { + const { data } = chunk.head.heading; + + chunk.name = unique(toFileName(headingLabel(data), data, module.api)); + chunk.path = `${module.path}/${chunk.name}`; + } + + const anchors = buildAnchorMap(chunks); + + chunks.forEach((chunk, index) => { + output.push(...buildChunkEntries(module, chunk, index, anchors)); + }); + } + + return output; +} diff --git a/packages/react/src/section-pages/index.mjs b/packages/react/src/section-pages/index.mjs new file mode 100644 index 000000000..b1043d110 --- /dev/null +++ b/packages/react/src/section-pages/index.mjs @@ -0,0 +1,48 @@ +'use strict'; + +import { generate } from './generate.mjs'; + +/** + * Section pages generator - splits each documented module into per-section pages. + * + * It takes the flattened metadata entries, and for every module adds a set of + * extra pages — one per section — alongside the untouched full page: + * + * ```text + * fs.html the complete module page, exactly as before + * fs/readFile.html one section of it (`fs.readFile()`) + * fs/Dir.html another (`Class: fs.Dir`, with all of its members) + * ``` + * + * It does not render anything itself. Instead it declares the generators it + * is delivered through: the orchestrator splices it between `metadata` and + * `jsx-ast` on the way to `html`, so the web pipeline renders the chunk pages + * just like any other page, and in front of `sitemap`, so they are listed. + * + * @type {import('./types').Generator} + */ +export default { + name: 'section-pages', + + description: + 'Splits each module into per-section pages, rendered by the html generator', + + dependsOn: '@doc-kit/core/metadata', + + dependent: [ + '@doc-kit/generator-react/html', + '@doc-kit/generator-react/sitemap', + ], + + defaultConfiguration: { + // Headings deeper than this never start a chunk of their own; they stay + // in the chunk of the closest heading above them. + maxDepth: 2, + // Modules (by `api` name) that are never split, on top of those that + // document no API entry at all (a landing page, a guide), which are never + // split anyway. + exclude: [], + }, + + generate, +}; diff --git a/packages/react/src/section-pages/types.d.ts b/packages/react/src/section-pages/types.d.ts new file mode 100644 index 000000000..bf57e1ee2 --- /dev/null +++ b/packages/react/src/section-pages/types.d.ts @@ -0,0 +1,28 @@ +import type { MetadataEntry } from '@doc-kit/core/generators/metadata/types'; + +export type Configuration = { + // Headings deeper than this never start a chunk of their own + maxDepth: number; + // Modules (by `api` name) that are never split + exclude: Array; +}; + +/** + * A group of consecutive metadata entries that become one chunk page, headed + * by the entry whose heading starts the section. + */ +export type Chunk = { + head: MetadataEntry; + // The original depth of the chunk's heading + depth: number; + entries: Array; + // The chunk's unique file name within its module, once assigned + name?: string; + // The chunk page's path (`/fs/readFile`), once assigned + path?: string; +}; + +export type Generator = GeneratorMetadata< + Configuration, + Generate, Promise>> +>; diff --git a/packages/react/src/section-pages/utils/links.mjs b/packages/react/src/section-pages/utils/links.mjs new file mode 100644 index 000000000..a0460d120 --- /dev/null +++ b/packages/react/src/section-pages/utils/links.mjs @@ -0,0 +1,51 @@ +'use strict'; + +import { posix } from 'node:path'; + +import { relative } from '@doc-kit/core/utils/url.mjs'; + +import { RELATIVE_URL } from '../constants.mjs'; + +/** + * Maps every heading anchor of a module to the chunk page it ends up on. + * Anchors that are not in the map (the module's introduction) only exist on + * the full page. + * + * @param {Array<{ path: string, entries: Array }>} chunks + * @returns {Map} Heading slug → chunk path + */ +export const buildAnchorMap = chunks => + new Map( + chunks.flatMap(({ path, entries }) => + entries.map(entry => [entry.heading.data.slug, path]) + ) + ); + +/** + * Re-targets a URL that was authored for the module page so it works from a + * chunk page: a fragment link follows its section to whichever page it landed + * on, and a relative URL is re-based from the module's directory to the + * chunk's. Any other URL is returned unchanged. + * + * @param {string} url + * @param {object} options + * @param {string} options.modulePath - The module page's path (`/fs`) + * @param {string} options.chunkPath - The chunk page's path (`/fs/readFile`) + * @param {Map} options.anchors - See {@link buildAnchorMap} + * @returns {string} + */ +export const rewriteUrl = (url, { modulePath, chunkPath, anchors }) => { + if (url.startsWith('#')) { + const target = anchors.get(url.slice(1)) ?? modulePath; + + return target === chunkPath + ? url + : `${relative(target, chunkPath)}.html${url}`; + } + + if (RELATIVE_URL.test(url)) { + return relative(posix.resolve(posix.dirname(modulePath), url), chunkPath); + } + + return url; +}; diff --git a/packages/react/src/section-pages/utils/naming.mjs b/packages/react/src/section-pages/utils/naming.mjs new file mode 100644 index 000000000..f4dbbada5 --- /dev/null +++ b/packages/react/src/section-pages/utils/naming.mjs @@ -0,0 +1,50 @@ +'use strict'; + +import { slug } from 'github-slugger'; + +import { SAFE_FILE_NAME } from '../constants.mjs'; + +/** + * Derives a chunk's file name (without extension) from its heading. + * + * API names keep their case so `fs.readFile()` lands at `fs/readFile.html`, + * with the module's own prefix dropped. Prose headings, and any API name that + * would not make a clean file name, are slugged instead (`Callback API` + * becomes `callback-api`). + * + * @param {string} label - The chunk's display label + * @param {import('@doc-kit/core/generators/metadata/types').HeadingData} heading - The chunk's heading data + * @param {string} api - The module's API name (e.g. `fs`) + * @returns {string} + */ +export const toFileName = (label, heading, api) => { + let name = label.replace(/^new\s+/, ''); + + if (name.startsWith(`${api}.`)) { + name = name.slice(api.length + 1); + } + + name = name.replace(/["']/g, ''); + + return heading.type && SAFE_FILE_NAME.test(name) ? name : slug(name); +}; + +/** + * Creates a function that makes file names unique within one module. + * Comparison is case-insensitive so the output is safe on case-insensitive + * file systems; a repeated name gets a numeric suffix (`close`, `close-2`). + * + * @returns {(name: string) => string} + */ +export const createUniqueNamer = () => { + const used = new Map(); + + return name => { + const key = name.toLowerCase(); + const count = (used.get(key) ?? 0) + 1; + + used.set(key, count); + + return count === 1 ? name : `${name}-${count}`; + }; +}; diff --git a/packages/react/src/section-pages/utils/split.mjs b/packages/react/src/section-pages/utils/split.mjs new file mode 100644 index 000000000..369c67967 --- /dev/null +++ b/packages/react/src/section-pages/utils/split.mjs @@ -0,0 +1,118 @@ +'use strict'; + +import { CHUNKABLE_TYPES, SELF_CONTAINED_TYPES } from '../constants.mjs'; + +/** + * Whether an entry documents an API entry rather than prose: the metadata + * generator typed its heading (a method, class, event, ...). + * + * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} entry + * @returns {boolean} + */ +const isApiEntry = entry => Boolean(entry.heading.data.type); + +/** + * Collects the entries whose section — the entry itself, or any heading + * nested below it, whether or not that heading gets a chunk of its own — + * documents at least one API entry. Sections that are pure prose (a module's + * "Introduction", the "Notes" at the end of a reference) are only worth + * reading in the context of the full page. + * + * @param {Array} entries - In document order + * @returns {Set} + */ +const findSectionsWithApi = entries => { + const withApi = new Set(); + + // The headings whose sections are still open at the current entry + const open = []; + + for (const entry of entries) { + const { depth } = entry.heading; + + while (open.length > 0 && open.at(-1).heading.depth >= depth) { + open.pop(); + } + + open.push(entry); + + if (isApiEntry(entry)) { + for (const section of open) { + withApi.add(section); + } + } + } + + return withApi; +}; + +/** + * Decides whether an entry's heading starts a new chunk. + * + * Every depth-2 heading does. Deeper headings only do when they document an + * API entry (a method, class, event, ...) at or above `maxDepth`, and when + * they are not part of a self-contained section such as a class. + * + * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} entry + * @param {import('../types').Chunk | undefined} current - The chunk being built + * @param {number} maxDepth + * @returns {boolean} + */ +const startsChunk = (entry, current, maxDepth) => { + const { depth, data } = entry.heading; + + if (depth > maxDepth) { + return false; + } + + const insideSelfContained = + current !== undefined && + depth > current.depth && + SELF_CONTAINED_TYPES.has(current.head.heading.data.type); + + if (insideSelfContained) { + return false; + } + + return depth === 2 || CHUNKABLE_TYPES.has(data.type); +}; + +/** + * Splits one module's entries (in document order) into chunks. + * + * The module's own depth-1 heading never becomes a chunk: the full page + * already covers it. Entries that precede the first chunk-starting heading, + * and sections that document no API entry at all, are therefore only present + * on the full page. + * + * @param {Array} entries + * @param {number} maxDepth + * @returns {Array} + */ +export const splitIntoChunks = (entries, maxDepth) => { + /** @type {Array} */ + const chunks = []; + + /** @type {import('../types').Chunk | undefined} */ + let current; + + for (const entry of entries) { + const { depth } = entry.heading; + + if (depth === 1) { + current = undefined; + continue; + } + + if (startsChunk(entry, current, maxDepth)) { + current = { head: entry, depth, entries: [entry] }; + chunks.push(current); + } else if (current) { + current.entries.push(entry); + } + } + + const withApi = findSectionsWithApi(entries); + + return chunks.filter(chunk => withApi.has(chunk.head)); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 569ca86aa..0df83080a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -247,6 +247,9 @@ importers: estree-util-to-js: specifier: ^2.0.0 version: 2.0.0 + github-slugger: + specifier: ^2.0.0 + version: 2.0.0 hast-util-to-string: specifier: ^3.0.1 version: 3.0.1 diff --git a/www/doc-kit.config.mjs b/www/doc-kit.config.mjs index 726d34f0e..ed9c4c090 100644 --- a/www/doc-kit.config.mjs +++ b/www/doc-kit.config.mjs @@ -14,6 +14,7 @@ const PUBLIC_GENERATORS = [ 'orama-db', 'llms-txt', 'sitemap', + 'section-pages', 'json-simple', 'legacy-html', 'legacy-html-all',