diff --git a/src/generators/jsx-ast/generate.mjs b/src/generators/jsx-ast/generate.mjs index bd268021..055e0efb 100644 --- a/src/generators/jsx-ast/generate.mjs +++ b/src/generators/jsx-ast/generate.mjs @@ -1,3 +1,5 @@ +import { jsx, toJs } from 'estree-util-to-js'; + import buildContent from './utils/buildContent.mjs'; import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; import { buildNotFoundPage } from './utils/synthetic/404.mjs'; @@ -7,30 +9,32 @@ import getConfig from '../../utils/configuration/index.mjs'; import { groupNodesByModule } from '../../utils/generators.mjs'; /** - * Builds JSX content for all configured synthetic pages. + * Builds the `{ head, entries }` page descriptors for all configured synthetic + * pages. The descriptors are cheap to build; the expensive `buildContent` step + * runs later in a worker (via `processChunk`), so the very large synthetic + * `all` page is never built on the main thread. * * @param {Array} input */ -const buildSyntheticEntries = async input => { +const buildSyntheticDescriptors = input => { const config = getConfig('jsx-ast'); - const descriptors = [ + return [ config.generateAllPage && buildAllPage(input), config.generateIndexPage && buildIndexPage(input), config.generateNotFoundPage && buildNotFoundPage(), ].filter(Boolean); - - return Promise.all( - descriptors.map(({ head, entries }) => buildContent(entries, head)) - ); }; /** * Process a chunk of items in a worker thread. - * Transforms metadata entries into JSX AST nodes. * - * Each item is a SlicedModuleInput containing the head node - * and all entries for that module - no need to recompute grouping. + * Each item is a `{ head, entries }` descriptor (one module, or a synthetic + * page). The JSX AST is built AND serialized to a code string here, inside the + * worker, so the heavy AST — most notably the giant `all` page, which + * concatenates every module — is dropped in the worker and never crosses back + * to or accumulates on the main thread. Only the much smaller code string and + * the page metadata are returned. * * @type {import('./types').Generator['processChunk']} */ @@ -42,14 +46,16 @@ export async function processChunk(slicedInput, itemIndices) { const content = await buildContent(entries, head); - results.push(content); + const { value: code } = toJs(content, { handlers: jsx }); + + results.push({ data: content.data, code }); } return results; } /** - * Generates a JSX AST + * Generates per-page JSX code from API metadata. * * @type {import('./types').Generator['generate']} */ @@ -60,18 +66,16 @@ export async function* generate(input, worker) { // Create sliced input: each item contains head + its module's entries // This avoids sending all 4700+ entries to every worker const groupedModules = groupNodesByModule(input); - const entries = getSortedHeadNodes(input).map(head => ({ + const descriptors = getSortedHeadNodes(input).map(head => ({ head, entries: groupedModules.get(head.api), })); - for await (const chunkResult of worker.stream(entries)) { - yield chunkResult; - } - - const syntheticEntries = await buildSyntheticEntries(moduleInput); + // Process the synthetic pages through the worker pool as well, so their + // (potentially enormous) content is built and converted off the main thread. + descriptors.push(...buildSyntheticDescriptors(moduleInput)); - if (syntheticEntries.length > 0) { - yield syntheticEntries; + for await (const chunkResult of worker.stream(descriptors)) { + yield chunkResult; } } diff --git a/src/generators/web/__tests__/generate.test.mjs b/src/generators/web/__tests__/generate.test.mjs index cde610ae..e5f70c76 100644 --- a/src/generators/web/__tests__/generate.test.mjs +++ b/src/generators/web/__tests__/generate.test.mjs @@ -1,11 +1,22 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { jsx, toJs } from 'estree-util-to-js'; + import { setConfig } from '../../../utils/configuration/index.mjs'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; import { generate } from '../generate.mjs'; +/** + * Converts a JSX AST entry into the `{ data, code }` shape `web` now consumes, + * mirroring the conversion the jsx-ast worker performs before streaming. + */ +const toCodeItem = content => ({ + data: content.data, + code: toJs(content, { handlers: jsx }).value, +}); + const createEntry = (api, name) => { const heading = { type: 'heading', @@ -39,10 +50,11 @@ describe('web generate', () => { const fs = createEntry('fs', 'File system'); const notFoundPage = buildNotFoundPage(); - const input = await Promise.all([ + const contents = await Promise.all([ buildContent([fs], fs), buildContent(notFoundPage.entries, notFoundPage.head), ]); + const input = contents.map(toCodeItem); const [fsPage, notFoundResult] = await generate(input); @@ -69,7 +81,7 @@ describe('web generate', () => { }; const fs = createEntry('fs', 'File system'); - const [fsPage] = await generate([await buildContent([fs], fs)]); + const [fsPage] = await generate([toCodeItem(await buildContent([fs], fs))]); assert.match(fsPage.html, /Custom project docs/); assert.match(fsPage.html, /https:\/\/example\.com\/og\.png/); diff --git a/src/generators/web/generate.mjs b/src/generators/web/generate.mjs index e5b19fb8..d3526900 100644 --- a/src/generators/web/generate.mjs +++ b/src/generators/web/generate.mjs @@ -4,15 +4,17 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { copyStaticAssets } from './utils/copying.mjs'; -import { processJSXEntries } from './utils/processing.mjs'; +import { createCodeConverter, processBundles } from './utils/processing.mjs'; import getConfig from '../../utils/configuration/index.mjs'; import { writeFile } from '../../utils/file.mjs'; /** - * Main generation function that processes JSX AST entries into web bundles. + * Main generation function that bundles per-page JSX code into web output. * - * Bundles all JSX AST entries in a single pass so shared component chunks and - * CSS are produced once. + * Receives `jsx-ast`'s output as `{ data, code }` items — the JSX AST was + * already serialized to `code` in the jsx-ast worker, so no AST is held here. + * Bundling and rendering then run once over the accumulated code, since shared + * component chunks, CSS, and the sidebar need every entry together. * * @type {import('./types').Generator['generate']} */ @@ -21,14 +23,30 @@ export async function generate(input) { const template = await readFile(config.templatePath, 'utf-8'); + const converter = createCodeConverter(); + + // Per-page metadata, in render order. Each item is already just + // `{ data, code }` — the heavy JSX AST was converted to `code` and discarded + // in the jsx-ast worker, so nothing large is held here. + const datas = []; + + for (const item of input) { + converter.add(item); + datas.push(item.data); + } + // Sidebar lists only the real module pages. - const sidebarEntries = input.filter(entry => entry.data.synthetic !== true); + const sidebarEntries = datas + .filter(data => data.synthetic !== true) + .map(data => ({ data })); - const { results, css, chunks } = await processJSXEntries( - input, + const { results, css, chunks } = await processBundles({ + serverCodeMap: converter.serverCodeMap, + clientCodeMap: converter.clientCodeMap, + datas, + sidebarEntries, template, - sidebarEntries - ); + }); if (config.output) { for (const { html, path } of results) { diff --git a/src/generators/web/index.mjs b/src/generators/web/index.mjs index 3695c864..2f4421ee 100644 --- a/src/generators/web/index.mjs +++ b/src/generators/web/index.mjs @@ -13,8 +13,11 @@ import { createLazyGenerator } from '../../utils/generators.mjs'; * - Client-side JavaScript with code splitting * - Bundled CSS styles * - * Note: This generator does NOT support streaming/chunked processing because - * processJSXEntries needs all entries together to generate code-split bundles. + * `jsx-ast` serializes each page's JSX AST to a `code` string inside its worker, + * so this generator only ever handles small `{ data, code }` items — the heavy + * ASTs (notably the giant `all` page) never reach the main thread. Bundling and + * rendering run once over the accumulated code, since code-splitting and the + * sidebar need every entry together. * * @type {import('./types').Generator} */ diff --git a/src/generators/web/utils/processing.mjs b/src/generators/web/utils/processing.mjs index 08aefdf1..84b6f53e 100644 --- a/src/generators/web/utils/processing.mjs +++ b/src/generators/web/utils/processing.mjs @@ -1,7 +1,6 @@ import { randomUUID } from 'node:crypto'; import { createRequire } from 'node:module'; -import { jsx, toJs } from 'estree-util-to-js'; import { transform } from 'lightningcss-wasm'; import bundleCode from './bundle.mjs'; @@ -80,31 +79,37 @@ export const buildHead = ({ meta = [], links = [], html = [] }) => ].join('\n '); /** - * Converts JSX AST entries to server and client JavaScript code. + * Creates an accumulator that wraps per-page JSX code into server and client + * programs one at a time. The JSX AST has already been serialized to a code + * string upstream (in the `jsx-ast` worker), so the heavy AST never reaches + * the main thread — only the code string and page metadata stream in here. * - * @param {Array} entries - JSX AST entries - * @param {function} buildServerProgram - Wraps code for server execution - * @param {function} buildClientProgram - Wraps code for client hydration - * @returns {{serverCodeMap: Map, clientCodeMap: Map}} + * @returns {{ add: (item: { data: import('../../metadata/types').MetadataEntry, code: string }) => void, serverCodeMap: Map, clientCodeMap: Map }} */ -function convertJSXToCode(entries, { buildServerProgram, buildClientProgram }) { +export function createCodeConverter() { + const { buildServerProgram, buildClientProgram } = createASTBuilder(); + const serverCodeMap = new Map(); const clientCodeMap = new Map(); - for (const entry of entries) { - const fileName = `${entry.data.api}.jsx`; - - // Convert AST to JavaScript string with JSX syntax - const { value: code } = toJs(entry, { handlers: jsx }); - - // Prepare code for server-side execution (wrapped for SSR) - serverCodeMap.set(fileName, buildServerProgram(code)); - - // Prepare code for client-side execution (wrapped for hydration) - clientCodeMap.set(fileName, buildClientProgram(code)); - } - - return { serverCodeMap, clientCodeMap }; + return { + /** + * Records the server/client programs for a single page's JSX code. + * + * @param {{ data: import('../../metadata/types').MetadataEntry, code: string }} item + */ + add: ({ data, code }) => { + const fileName = `${data.api}.jsx`; + + // Prepare code for server-side execution (wrapped for SSR) + serverCodeMap.set(fileName, buildServerProgram(code)); + + // Prepare code for client-side execution (wrapped for hydration) + clientCodeMap.set(fileName, buildClientProgram(code)); + }, + serverCodeMap, + clientCodeMap, + }; } /** @@ -140,32 +145,34 @@ async function executeServerCode(serverCodeMap, requireFn, virtualImports) { } /** - * Processes JSX AST entries into complete HTML pages, client JS bundles, and CSS. + * Bundles pre-converted JSX code into complete HTML pages, client JS bundles, + * and CSS. Conversion (JSX AST → code) happens upstream via + * {@link createCodeConverter} so the heavy ASTs are already discarded; this + * step needs every entry together for code-splitting and the shared sidebar. * - * @param {Array} entries - The JSX AST entries to process. - * @param {string} template - The HTML template string for the output pages. - * @param {Array<{ data: import('../../metadata/types').MetadataEntry }>} [sidebarEntries] - Entries used to build the sidebar page list. Defaults to `entries`. Pass the full set when rendering a subset (e.g. the `all` page) so the sidebar still links to every module. + * @param {object} params + * @param {Map} params.serverCodeMap - Server-side code per page. + * @param {Map} params.clientCodeMap - Client-side code per page. + * @param {Array} params.datas - Per-page metadata, in render order. + * @param {Array<{ data: import('../../metadata/types').MetadataEntry }>} params.sidebarEntries - Entries used to build the sidebar page list (real module pages only). + * @param {string} params.template - The HTML template string for the output pages. */ -export async function processJSXEntries( - entries, +export async function processBundles({ + serverCodeMap, + clientCodeMap, + datas, + sidebarEntries, template, - sidebarEntries = entries -) { +}) { const config = getConfig('web'); - const astBuilders = createASTBuilder(); const requireFn = createRequire(import.meta.url); const virtualImports = { '#theme/config': createConfigSource(sidebarEntries), ...config.virtualImports, }; - // Step 1: Convert JSX AST to JavaScript - const { serverCodeMap, clientCodeMap } = convertJSXToCode( - entries, - astBuilders - ); - // Step 2: Bundle server and client code in parallel - // Both need all entries for code-splitting, but are independent of each other + // Bundle server and client code in parallel. Both need all entries for + // code-splitting, but are independent of each other. const [serverBundle, clientBundle] = await Promise.all([ executeServerCode(serverCodeMap, requireFn, virtualImports), bundleCode(clientCodeMap, virtualImports), @@ -181,9 +188,9 @@ export async function processJSXEntries( // template authors avoid nested template-literal escaping. const head = buildHead(config.head); - // Step 3: Render final HTML pages + // Render final HTML pages const results = await Promise.all( - entries.map(async ({ data }) => { + datas.map(async data => { const root = resolvePageRoot(data); // Replace template placeholders with actual content