Backport v14: Retry manifest file loading only in dev mode (#73900) (#74282)

Backports:
- #73900

Co-authored-by: Hendrik Liebau <[email protected]>

Author: ztanner

packages/next/src/build/utils.ts

@@ -1569,6 +1569,7 @@ export async function isPageStatic({
           distDir,
           page: originalAppPath || page,
           isAppPath: pageType === 'app',
+          isDev: false,
         })
       }
       const Comp = componentsResult.Component as NextComponentType | undefined
@@ -1802,6 +1803,7 @@ export async function hasCustomGetInitialProps({
     distDir,
     page: page,
     isAppPath: false,
+    isDev: false,
   })
   let mod = components.ComponentMod
 
@@ -1828,6 +1830,7 @@ export async function getDefinedNamedExports({
     distDir,
     page: page,
     isAppPath: false,
+    isDev: false,
   })
 
   return Object.keys(components.ComponentMod).filter((key) => {

packages/next/src/export/worker.ts

@@ -257,6 +257,7 @@ async function exportPageImpl(
       distDir,
       page,
       isAppPath: isAppDir,
+      isDev: false,
     })
 
     const renderOpts: WorkerRenderOpts = {

packages/next/src/server/dev/static-paths-worker.ts

@@ -71,6 +71,7 @@ export async function loadStaticPaths({
     // In `pages/`, the page is the same as the pathname.
     page: page || pathname,
     isAppPath,
+    isDev: true,
   })
 
   if (!components.getStaticPaths && !isAppPath) {

packages/next/src/server/load-components.ts

@@ -110,12 +110,13 @@ export async function evalManifestWithRetries<T extends object>(
 
 async function loadClientReferenceManifest(
   manifestPath: string,
-  entryName: string
+  entryName: string,
+  attempts?: number
 ) {
   try {
     const context = await evalManifestWithRetries<{
       __RSC_MANIFEST: { [key: string]: ClientReferenceManifest }
-    }>(manifestPath)
+    }>(manifestPath, attempts)
     return context.__RSC_MANIFEST[entryName]
   } catch (err) {
     return undefined
@@ -126,10 +127,12 @@ async function loadComponentsImpl<N = any>({
   distDir,
   page,
   isAppPath,
+  isDev,
 }: {
   distDir: string
   page: string
   isAppPath: boolean
+  isDev: boolean
 }): Promise<LoadComponentsReturnType<N>> {
   let DocumentMod = {}
   let AppMod = {}
@@ -144,16 +147,26 @@ async function loadComponentsImpl<N = any>({
   const hasClientManifest =
     isAppPath && (page.endsWith('/page') || page === UNDERSCORE_NOT_FOUND_ROUTE)
 
+  // In dev mode we retry loading a manifest file to handle a race condition
+  // that can occur while app and pages are compiling at the same time, and the
+  // build-manifest is still being written to disk while an app path is
+  // attempting to load.
+  const manifestLoadAttempts = isDev ? 3 : 1
+
   // Load the manifest files first
   const [
     buildManifest,
     reactLoadableManifest,
     clientReferenceManifest,
     serverActionsManifest,
   ] = await Promise.all([
-    loadManifestWithRetries<BuildManifest>(join(distDir, BUILD_MANIFEST)),
+    loadManifestWithRetries<BuildManifest>(
+      join(distDir, BUILD_MANIFEST),
+      manifestLoadAttempts
+    ),
     loadManifestWithRetries<ReactLoadableManifest>(
-      join(distDir, REACT_LOADABLE_MANIFEST)
+      join(distDir, REACT_LOADABLE_MANIFEST),
+      manifestLoadAttempts
     ),
     hasClientManifest
       ? loadClientReferenceManifest(
@@ -163,12 +176,14 @@ async function loadComponentsImpl<N = any>({
             'app',
             page.replace(/%5F/g, '_') + '_' + CLIENT_REFERENCE_MANIFEST + '.js'
           ),
-          page.replace(/%5F/g, '_')
+          page.replace(/%5F/g, '_'),
+          manifestLoadAttempts
         )
       : undefined,
     isAppPath
       ? loadManifestWithRetries<ActionManifest>(
-          join(distDir, 'server', SERVER_REFERENCE_MANIFEST + '.json')
+          join(distDir, 'server', SERVER_REFERENCE_MANIFEST + '.json'),
+          manifestLoadAttempts
         ).catch(() => null)
       : null,
   ])

packages/next/src/server/next-server.ts

@@ -179,11 +179,14 @@ export default class NextNodeServer extends BaseServer {
     req: IncomingMessage,
     res: ServerResponse
   ) => void
+  private isDev: boolean
 
   constructor(options: Options) {
     // Initialize super class
     super(options)
 
+    this.isDev = options.dev ?? false
+
     /**
      * This sets environment variable to be used at the time of SSR by head.tsx.
      * Using this from process.env allows targeting SSR by calling
@@ -220,11 +223,13 @@ export default class NextNodeServer extends BaseServer {
         distDir: this.distDir,
         page: '/_document',
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
       loadComponents({
         distDir: this.distDir,
         page: '/_app',
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
     }
 
@@ -281,11 +286,17 @@ export default class NextNodeServer extends BaseServer {
         distDir: this.distDir,
         page,
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
     }
 
     for (const page of Object.keys(appPathsManifest || {})) {
-      await loadComponents({ distDir: this.distDir, page, isAppPath: true })
+      await loadComponents({
+        distDir: this.distDir,
+        page,
+        isAppPath: true,
+        isDev: this.isDev,
+      })
         .then(async ({ ComponentMod }) => {
           const webpackRequire = ComponentMod.__next_app__.require
           if (webpackRequire?.m) {
@@ -758,6 +769,7 @@ export default class NextNodeServer extends BaseServer {
           distDir: this.distDir,
           page: pagePath,
           isAppPath,
+          isDev: this.isDev,
         })
 
         if (