fix(prerender): ensure failed prerenders are retried

packages/start-plugin-core/src/prerender.ts

@@ -89,6 +89,10 @@ export async function prerender({
     const concurrency = startConfig.prerender?.concurrency ?? os.cpus().length
     logger.info(`Concurrency: ${concurrency}`)
     const queue = new Queue({ concurrency })
+    const taskErrors: Array<unknown> = []
+    queue.onError((error) => {
+      taskErrors.push(error)
+    })
     const routerBasePath = joinURL('/', startConfig.router.basepath ?? '')
     const routerBaseUrl = new URL(routerBasePath, 'http://localhost')
 
@@ -106,6 +110,10 @@ export async function prerender({
 
     await queue.start()
 
+    if (taskErrors.length > 0) {
+      throw taskErrors[0]
+    }
+
     return Array.from(prerendered)
 
     function addCrawlPageTask(page: Page) {
@@ -126,105 +134,109 @@ export async function prerender({
         return
       }
 
+      // Errors are collected via queue.onError() and rethrown once the queue
+      // settles. Suppress intermediate errors by catching here.
+      queue.add(() => renderPage(page)).catch(() => {})
+    }
+
+    async function renderPage(page: Page): Promise<void> {
       const prerenderOptions = {
         ...startConfig.prerender,
         ...page.prerender,
       }
 
-      queue.add(async () => {
-        logger.info(`Crawling: ${page.path}`)
-        const retries = retriesByPath.get(page.path) || 0
-
-        try {
-          const res = await requestWithRedirects(
-            withTrailingSlash(withBase(page.path, routerBasePath)),
-            {
-              headers: {
-                ...(prerenderOptions.headers ?? {}),
-              },
-            },
-            prerenderOptions.maxRedirects,
-          )
+      logger.info(`Crawling: ${page.path}`)
+      const retries = retriesByPath.get(page.path) || 0
 
-          if (!res.ok) {
-            if (isRedirectResponse(res)) {
-              logger.warn(`Max redirects reached for ${page.path}`)
-            }
+      try {
+        const res = await requestWithRedirects(
+          withTrailingSlash(withBase(page.path, routerBasePath)),
+          {
+            headers: {
+              ...(prerenderOptions.headers ?? {}),
+            },
+          },
+          prerenderOptions.maxRedirects,
+        )
 
-            throw new Error(`Failed to fetch ${page.path}: ${res.statusText}`, {
-              cause: res,
-            })
+        if (!res.ok) {
+          if (isRedirectResponse(res)) {
+            logger.warn(`Max redirects reached for ${page.path}`)
           }
 
-          const cleanPagePath = (
-            prerenderOptions.outputPath || page.path
-          ).split(/[?#]/)[0]!
-
-          const contentType = res.headers.get('content-type') || ''
-          const isImplicitHTML =
-            !cleanPagePath.endsWith('.html') && contentType.includes('html')
-
-          const routeWithIndex = cleanPagePath.endsWith('/')
-            ? cleanPagePath + 'index'
-            : cleanPagePath
-
-          const isSpaShell =
-            startConfig.spa?.prerender.outputPath === cleanPagePath
-
-          let htmlPath: string
-          if (isSpaShell) {
-            htmlPath = cleanPagePath + '.html'
-          } else if (
-            cleanPagePath.endsWith('/') ||
-            (prerenderOptions.autoSubfolderIndex ?? true)
-          ) {
-            htmlPath = joinURL(cleanPagePath, 'index.html')
-          } else {
-            htmlPath = cleanPagePath + '.html'
-          }
+          throw new Error(`Failed to fetch ${page.path}: ${res.statusText}`, {
+            cause: res,
+          })
+        }
 
-          const filename = withoutBase(
-            isImplicitHTML ? htmlPath : routeWithIndex,
-            routerBasePath,
-          )
+        const cleanPagePath = (
+          prerenderOptions.outputPath || page.path
+        ).split(/[?#]/)[0]!
+
+        const contentType = res.headers.get('content-type') || ''
+        const isImplicitHTML =
+          !cleanPagePath.endsWith('.html') && contentType.includes('html')
+
+        const routeWithIndex = cleanPagePath.endsWith('/')
+          ? cleanPagePath + 'index'
+          : cleanPagePath
+
+        const isSpaShell =
+          startConfig.spa?.prerender.outputPath === cleanPagePath
+
+        let htmlPath: string
+        if (isSpaShell) {
+          htmlPath = cleanPagePath + '.html'
+        } else if (
+          cleanPagePath.endsWith('/') ||
+          (prerenderOptions.autoSubfolderIndex ?? true)
+        ) {
+          htmlPath = joinURL(cleanPagePath, 'index.html')
+        } else {
+          htmlPath = cleanPagePath + '.html'
+        }
 
-          const html = await res.text()
-          const filepath = path.join(outputDir, filename)
+        const filename = withoutBase(
+          isImplicitHTML ? htmlPath : routeWithIndex,
+          routerBasePath,
+        )
 
-          await fsp.mkdir(path.dirname(filepath), {
-            recursive: true,
-          })
+        const html = await res.text()
+        const filepath = path.join(outputDir, filename)
 
-          await fsp.writeFile(filepath, html)
+        await fsp.mkdir(path.dirname(filepath), {
+          recursive: true,
+        })
 
-          prerendered.add(page.path)
+        await fsp.writeFile(filepath, html)
 
-          const newPage = await prerenderOptions.onSuccess?.({ page, html })
+        prerendered.add(page.path)
 
-          if (newPage) {
-            Object.assign(page, newPage)
-          }
+        const newPage = await prerenderOptions.onSuccess?.({ page, html })
 
-          if (prerenderOptions.crawlLinks ?? true) {
-            const links = extractLinks(html)
-            for (const link of links) {
-              addCrawlPageTask({ path: link, fromCrawl: true })
-            }
-          }
-        } catch (error) {
-          if (retries < (prerenderOptions.retryCount ?? 0)) {
-            const retryDelay = normalizeRetryDelay(prerenderOptions.retryDelay)
-            logger.warn(
-              `Encountered error, retrying: ${page.path} in ${retryDelay}ms`,
-            )
-            await new Promise((resolve) => setTimeout(resolve, retryDelay))
-            retriesByPath.set(page.path, retries + 1)
-            addCrawlPageTask(page)
-          } else if (prerenderOptions.failOnError ?? true) {
-            throw error
+        if (newPage) {
+          Object.assign(page, newPage)
+        }
+
+        if (prerenderOptions.crawlLinks ?? true) {
+          const links = extractLinks(html)
+          for (const link of links) {
+            addCrawlPageTask({ path: link, fromCrawl: true })
           }
         }
-      })
+      } catch (error) {
+        if (retries < (prerenderOptions.retryCount ?? 0)) {
+          const retryDelay = normalizeRetryDelay(prerenderOptions.retryDelay)
+          logger.warn(
+            `Encountered error, retrying: ${page.path} in ${retryDelay}ms`,
+          )
+          await new Promise((resolve) => setTimeout(resolve, retryDelay))
+          retriesByPath.set(page.path, retries + 1)
+          queue.add(() => renderPage(page)).catch(() => {})
+        } else if (prerenderOptions.failOnError ?? true) {
+          throw error
+        }
+      }
     }
   }
 

packages/start-plugin-core/tests/prerender-retry.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, it, vi } from 'vitest'
+import { prerender } from '../src/prerender'
+import type {
+  TanStackStartInputConfig,
+  TanStackStartOutputConfig,
+} from '../src/schema'
+import type * as fsModule from 'node:fs'
+import type * as utilsModule from '../src/utils'
+
+vi.mock('../src/utils', async () => {
+  const actual = await vi.importActual<typeof utilsModule>('../src/utils')
+  return {
+    ...actual,
+    createLogger: () => ({ info: () => {}, warn: () => {}, error: () => {} }),
+  }
+})
+
+vi.mock('node:fs', async () => {
+  const actual = await vi.importActual<typeof fsModule>('node:fs')
+  return {
+    ...actual,
+    promises: {
+      ...actual.promises,
+      mkdir: vi.fn().mockResolvedValue(undefined),
+      writeFile: vi.fn().mockResolvedValue(undefined),
+    },
+  }
+})
+
+function okResponse(body = '<html></html>') {
+  return new Response(body, {
+    status: 200,
+    headers: { 'content-type': 'text/html' },
+  })
+}
+
+function failingResponse() {
+  return new Response('', {
+    status: 500,
+    statusText: 'Internal Server Error',
+  })
+}
+
+function makeStartConfig(
+  pagePath: string,
+  prerenderOverrides: Record<string, unknown> = {},
+): TanStackStartOutputConfig {
+  const config: TanStackStartInputConfig = {
+    importProtection: {},
+    prerender: {
+      enabled: true,
+      autoStaticPathsDiscovery: false,
+      concurrency: 1,
+      crawlLinks: false,
+      ...prerenderOverrides,
+    },
+    pages: [{ path: pagePath }],
+    router: { basepath: '' },
+    spa: {
+      enabled: false,
+      prerender: {
+        outputPath: '/_shell',
+        crawlLinks: false,
+        retryCount: 0,
+        enabled: true,
+      },
+    },
+  }
+
+  return config as unknown as TanStackStartOutputConfig
+}
+
+describe('prerender retry behaviour', () => {
+  it('re-fetches the page on retry', async () => {
+    const requestMock = vi
+      .fn()
+      .mockResolvedValueOnce(failingResponse())
+      .mockResolvedValueOnce(failingResponse())
+      .mockResolvedValueOnce(okResponse())
+
+    const handler = {
+      getClientOutputDirectory: () => '/client',
+      request: requestMock,
+    }
+
+    const startConfig = makeStartConfig('/about', {
+      retryCount: 2,
+      retryDelay: 0,
+    })
+
+    await expect(prerender({ startConfig, handler })).resolves.not.toThrow()
+
+    // One initial attempt plus two retries
+    expect(requestMock).toHaveBeenCalledTimes(3)
+  })
+
+  it('stops after retryCount attempts', async () => {
+    const requestMock = vi.fn().mockResolvedValue(failingResponse())
+
+    const handler = {
+      getClientOutputDirectory: () => '/client',
+      request: requestMock,
+    }
+
+    const startConfig = makeStartConfig('/broken', {
+      retryCount: 2,
+      retryDelay: 0,
+      failOnError: false,
+    })
+
+    await expect(prerender({ startConfig, handler })).resolves.not.toThrow()
+
+    // One initial attempt plus two retries
+    expect(requestMock).toHaveBeenCalledTimes(3)
+  })
+
+  it('rejects once retries are exhausted with the default failOnError behaviour', async () => {
+    const requestMock = vi.fn().mockResolvedValue(failingResponse())
+
+    const handler = {
+      getClientOutputDirectory: () => '/client',
+      request: requestMock,
+    }
+
+    const startConfig = makeStartConfig('/broken', {
+      retryCount: 2,
+      retryDelay: 0,
+    })
+
+    await expect(prerender({ startConfig, handler })).rejects.toThrow(
+      /Failed to fetch/i,
+    )
+
+    // One initial attempt plus two retries
+    expect(requestMock).toHaveBeenCalledTimes(3)
+  })
+})