[turbopack] Make it possible to synchronously access native bindings (#85787)

## Enable synchronous access to native bindings

## What?
This PR refactors the SWC bindings loading mechanism to support synchronous access to bindings after they've been loaded. It introduces new functions `installBindings()` and `getBindingsSync()` to properly manage the bindings lifecycle.

`installBindings` is called early by tools that rely on them.

There were of course  a few complexities:

* webpack loaders need to defensively install bindings to support being run by webpack
* the hacky `loadNativeSync` method used by `transformSync` remains required due to the jest-transformer API (i didn't see a clear initialization location).
* the `experimental.useWasmBinary` configuration is broken by design
   * if the config is `next.config.ts` when we load SWC to transpile it, but that requires the native bindings, so we might load native bindings even when you ask for wasm.  I added a warning message for this case and the workaround is to use the node native transpilation support.
   * this should probably be a command line flag if it is important
* we need to lazily load the `swc/index.ts` module since it transitively loads `react` which reads `NODE_ENV` and various next entrypoints might modify that.

## Why?

This makes it clearer that we are correctly applying the `useWasmBinary` option and can simplify some code paths that now don't need to be `async`.  This will be important in a future PR where i move some parts of issue formatting into rust/wasm.  Having those APIs be async will decrease their usability.

Author: lukesandberg

packages/next/errors.json

@@ -903,5 +903,7 @@
   "902": "Invalid handler fields configured for \"cacheHandlers\":\\n%s",
   "903": "The file \"%s\" must export a function, either as a default export or as a named \"%s\" export.\\nThis function is what Next.js runs for every request handled by this %s.\\n\\nWhy this happens:\\n%s- The file exists but doesn't export a function.\\n- The export is not a function (e.g., an object or constant).\\n- There's a syntax error preventing the export from being recognized.\\n\\nTo fix it:\\n- Ensure this file has either a default or \"%s\" function export.\\n\\nLearn more: https://nextjs.org/docs/messages/middleware-to-proxy",
   "904": "The file \"%s\" must export a function, either as a default export or as a named \"%s\" export.",
-  "905": "Page \"%s\" cannot use \\`export const unstable_prefetch = ...\\` without enabling \\`cacheComponents\\`."
+  "905": "Page \"%s\" cannot use \\`export const unstable_prefetch = ...\\` without enabling \\`cacheComponents\\`.",
+  "906": "Bindings not loaded yet, but they are being loaded, did you forget to await?",
+  "907": "bindings not loaded yet.  Either call `loadBindings` to wait for them to be available or ensure that `installBindings` has already been called."
 }

packages/next/src/build/babel/loader/get-config.ts

@@ -17,6 +17,7 @@ import {
 } from './util'
 import * as Log from '../../output/log'
 import { isReactCompilerRequired } from '../../swc'
+import { installBindings } from '../../swc/install-bindings'
 
 /**
  * An internal (non-exported) type used by babel.
@@ -570,6 +571,10 @@ export default async function getConfig(
     inputSourceMap?: SourceMap | undefined
   }
 ): Promise<ResolvedBabelConfig | null> {
+  // Install bindings early so they are definitely available to the loader.
+  // When run by webpack in next this is already done with correct configuration so this is a no-op.
+  // In turbopack loaders are run in a subprocess so it may or may not be done.
+  await installBindings()
   const cacheCharacteristics = await getCacheCharacteristics(
     loaderOptions,
     source,

packages/next/src/build/index.ts

@@ -151,6 +151,7 @@ import type { NextError } from '../lib/is-error'
 import { isEdgeRuntime } from '../lib/is-edge-runtime'
 import { recursiveCopy } from '../lib/recursive-copy'
 import { lockfilePatchPromise, teardownTraceSubscriber } from './swc'
+import { installBindings } from './swc/install-bindings'
 import { getNamedRouteRegex } from '../shared/lib/router/utils/route-regex'
 import { getFilesInDir } from '../lib/get-files-in-dir'
 import { eventSwcPlugins } from '../telemetry/events/swc-plugins'
@@ -964,6 +965,8 @@ export default async function build(
       // Reading the config can modify environment variables that influence the bundler selection.
       bundler = finalizeBundlerFromConfig(bundler)
       nextBuildSpan.setAttribute('bundler', getBundlerForTelemetry(bundler))
+      // Install the native bindings early so we can have synchronous access later.
+      await installBindings(config.experimental?.useWasmBinary)
 
       process.env.NEXT_DEPLOYMENT_ID = config.deploymentId || ''
       NextBuildContext.config = config

packages/next/src/build/jest/jest.ts

@@ -5,7 +5,8 @@ import { PHASE_TEST } from '../../shared/lib/constants'
 import loadJsConfig from '../load-jsconfig'
 import * as Log from '../output/log'
 import { findPagesDir } from '../../lib/find-pages-dir'
-import { loadBindings, lockfilePatchPromise } from '../swc'
+import { lockfilePatchPromise } from '../swc'
+import { installBindings } from '../swc/install-bindings'
 import type { JestTransformerConfig } from '../swc/jest-transformer'
 import type { Config } from '@jest/types'
 
@@ -96,7 +97,7 @@ export default function nextJest(options: { dir?: string } = {}) {
           : customJestConfig) ?? {}
 
       // eagerly load swc bindings instead of waiting for transform calls
-      await loadBindings(nextConfig?.experimental?.useWasmBinary)
+      await installBindings(nextConfig?.experimental?.useWasmBinary)
 
       if (lockfilePatchPromise.cur) {
         await lockfilePatchPromise.cur

packages/next/src/build/load-entrypoint.ts

@@ -1,6 +1,6 @@
 import fs from 'fs/promises'
 import path from 'path'
-import { loadBindings } from './swc'
+import { getBindingsSync } from './swc'
 
 // NOTE: this should be updated if this loader file is moved.
 const PACKAGE_ROOT = path.normalize(path.join(__dirname, '../..'))
@@ -39,14 +39,12 @@ export async function loadEntrypoint(
   injections?: Record<string, string>,
   imports?: Record<string, string | null>
 ): Promise<string> {
-  let bindings = await loadBindings()
-
   const templatePath = path.resolve(
     path.join(TEMPLATES_ESM_FOLDER, `${entrypoint}.js`)
   )
   let content = await fs.readFile(templatePath)
 
-  return bindings.expandNextJsTemplate(
+  return getBindingsSync().expandNextJsTemplate(
     content,
     // Ensure that we use unix-style path separators for the import paths
     path.join(TEMPLATE_SRC_FOLDER, `${entrypoint}.js`).replace(/\\/g, '/'),

packages/next/src/build/lockfile.ts

@@ -1,5 +1,6 @@
 import { bold, cyan } from '../lib/picocolors'
 import * as Log from './output/log'
+import { getBindingsSync } from './swc'
 
 import type { Binding, Lockfile as NativeLockfile } from './swc/types'
 
@@ -56,16 +57,11 @@ export class Lockfile {
    * - If we fail to acquire the lock, we return `undefined`.
    * - If we're on wasm, this always returns a dummy `Lockfile` object.
    */
-  static async tryAcquire(
+  static tryAcquire(
     path: string,
     unlockOnExit: boolean = true
-  ): Promise<Lockfile | undefined> {
-    const { loadBindings } = require('./swc') as typeof import('./swc')
-    // Ideally we could provide a sync version of `tryAcquire`, but
-    // `loadBindings` is async. We're okay with skipping async-loaded wasm
-    // bindings and the internal `loadNative` function is synchronous, but it
-    // lacks some checks that `loadBindings` has.
-    const bindings = await loadBindings()
+  ): Lockfile | undefined {
+    const bindings = getBindingsSync()
     if (bindings.isWasm) {
       Log.info(
         `Skipping creating a lockfile at ${cyan(path)} because we're using WASM bindings`
@@ -74,7 +70,7 @@ export class Lockfile {
     } else {
       let nativeLockfile
       try {
-        nativeLockfile = await bindings.lockfileTryAcquire(path)
+        nativeLockfile = bindings.lockfileTryAcquireSync(path)
       } catch (e) {
         // this happens if there's an IO error (e.g. `ENOENT`), which is
         // different than if we just didn't acquire the lock
@@ -123,7 +119,7 @@ export class Lockfile {
     const startMs = Date.now()
     let lockfile
     while (Date.now() - startMs < MAX_RETRY_MS) {
-      lockfile = await Lockfile.tryAcquire(path, unlockOnExit)
+      lockfile = Lockfile.tryAcquire(path, unlockOnExit)
       if (lockfile !== undefined) break
       await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS))
     }

packages/next/src/build/next-config-ts/transpile-config.ts

@@ -177,8 +177,9 @@ async function handleCJS({
     const nextConfigString = await readFile(nextConfigPath, 'utf8')
     // lazy require swc since it loads React before even setting NODE_ENV
     // resulting loading Development React on Production
-    const { transform } = require('../swc') as typeof import('../swc')
-    const { code } = await transform(nextConfigString, swcOptions)
+    const { loadBindings } = require('../swc') as typeof import('../swc')
+    const bindings = await loadBindings()
+    const { code } = await bindings.transform(nextConfigString, swcOptions)
 
     // register require hook only if require exists
     if (code.includes('require(')) {
@@ -187,7 +188,21 @@ async function handleCJS({
     }
 
     // filename & extension don't matter here
-    return requireFromString(code, resolve(cwd, 'next.config.compiled.js'))
+    const config = requireFromString(
+      code,
+      resolve(cwd, 'next.config.compiled.js')
+    )
+    // At this point we have already loaded the bindings without this configuration setting due to the `transform` call above.
+    // Possibly we fell back to wasm in which case, it all works out but if not we need to warn
+    // that the configuration was ignored.
+    if (config?.experimental?.useWasmBinary && !bindings.isWasm) {
+      warn(
+        'Using a next.config.ts file is incompatible with `experimental.useWasmBinary` unless ' +
+          '`--experimental-next-config-strip-types` is also passed.\nSetting `useWasmBinary` to `false'
+      )
+      config.experimental.useWasmBinary = false
+    }
+    return config
   } catch (error) {
     throw error
   } finally {

packages/next/src/build/swc/index.ts

@@ -160,21 +160,33 @@ let lastNativeBindingsLoadErrorCode:
   | 'unsupported_target'
   | string
   | undefined = undefined
-// Used to cache calls to `loadBindings`
-let pendingBindings: Promise<Binding>
-// some things call `loadNative` directly instead of `loadBindings`... Cache calls to that
-// separately.
-let nativeBindings: Binding
-// can allow hacky sync access to bindings for loadBindingsSync
-let wasmBindings: Binding
+// Used to cache racing calls to `loadBindings`
+let pendingBindings: Promise<Binding> | undefined
+// The cached loaded bindings
+let loadedBindings: Binding | undefined = undefined
 let downloadWasmPromise: any
 let swcTraceFlushGuard: any
 let downloadNativeBindingsPromise: Promise<void> | undefined = undefined
 
 export const lockfilePatchPromise: { cur?: Promise<void> } = {}
 
+/** Access the native bindings which should already have been loaded via `installBindings.  Throws if they are not available. */
+export function getBindingsSync(): Binding {
+  if (!loadedBindings) {
+    if (pendingBindings) {
+      throw new Error(
+        'Bindings not loaded yet, but they are being loaded, did you forget to await?'
+      )
+    }
+    throw new Error(
+      'bindings not loaded yet.  Either call `loadBindings` to wait for them to be available or ensure that `installBindings` has already been called.'
+    )
+  }
+  return loadedBindings
+}
+
 /**
- * Attempts to load a native or wasm binding.
+ * Loads the native or wasm binding.
  *
  * By default, this first tries to use a native binding, falling back to a wasm binding if that
  * fails.
@@ -184,6 +196,9 @@ export const lockfilePatchPromise: { cur?: Promise<void> } = {}
 export async function loadBindings(
   useWasmBinary: boolean = false
 ): Promise<Binding> {
+  if (loadedBindings) {
+    return loadedBindings
+  }
   if (pendingBindings) {
     return pendingBindings
   }
@@ -281,7 +296,9 @@ export async function loadBindings(
 
     logLoadFailure(attempts, true)
   })
-  return pendingBindings
+  loadedBindings = await pendingBindings
+  pendingBindings = undefined
+  return loadedBindings
 }
 
 async function tryLoadNativeWithFallback(attempts: Array<string>) {
@@ -363,12 +380,6 @@ function loadBindingsSync() {
     attempts = attempts.concat(a)
   }
 
-  // HACK: we can leverage the wasm bindings if they are already loaded
-  // this may introduce race conditions
-  if (wasmBindings) {
-    return wasmBindings
-  }
-
   logLoadFailure(attempts)
   throw new Error('Failed to load bindings', { cause: attempts })
 }
@@ -1182,7 +1193,7 @@ async function loadWasm(importPath = '') {
 
   // Note wasm binary does not support async intefaces yet, all async
   // interface coereces to sync interfaces.
-  wasmBindings = {
+  let wasmBindings = {
     css: {
       lightning: {
         transform: function (_options: any) {
@@ -1312,8 +1323,8 @@ async function loadWasm(importPath = '') {
  * wasm fallback.
  */
 function loadNative(importPath?: string) {
-  if (nativeBindings) {
-    return nativeBindings
+  if (loadedBindings) {
+    return loadedBindings
   }
 
   if (process.env.NEXT_TEST_WASM) {
@@ -1379,7 +1390,7 @@ function loadNative(importPath?: string) {
   }
 
   if (bindings) {
-    nativeBindings = {
+    loadedBindings = {
       isWasm: false,
       transform(src: string, options: any) {
         const isModule =
@@ -1518,7 +1529,7 @@ function loadNative(importPath?: string) {
         return bindings.lockfileUnlockSync(lockfile)
       },
     }
-    return nativeBindings
+    return loadedBindings
   }
 
   throw attempts
@@ -1539,54 +1550,40 @@ function toBuffer(t: any) {
   return Buffer.from(JSON.stringify(t))
 }
 
-export async function isWasm(): Promise<boolean> {
-  let bindings = await loadBindings()
-  return bindings.isWasm
-}
-
 export async function transform(src: string, options?: any): Promise<any> {
-  let bindings = await loadBindings()
+  let bindings = getBindingsSync()
   return bindings.transform(src, options)
 }
 
+/** Synchronously transforms the source and loads the native bindings. */
 export function transformSync(src: string, options?: any): any {
-  let bindings = loadBindingsSync()
+  const bindings = loadBindingsSync()
   return bindings.transformSync(src, options)
 }
 
-export async function minify(
+export function minify(
   src: string,
   options: any
 ): Promise<{ code: string; map: any }> {
-  let bindings = await loadBindings()
+  const bindings = getBindingsSync()
   return bindings.minify(src, options)
 }
 
-export async function isReactCompilerRequired(
-  filename: string
-): Promise<boolean> {
-  let bindings = await loadBindings()
+export function isReactCompilerRequired(filename: string): Promise<boolean> {
+  const bindings = getBindingsSync()
   return bindings.reactCompiler.isReactCompilerRequired(filename)
 }
 
 export async function parse(src: string, options: any): Promise<any> {
-  let bindings = await loadBindings()
-  let parserOptions = getParserOptions(options)
-  return bindings
-    .parse(src, parserOptions)
-    .then((astStr: any) => JSON.parse(astStr))
+  const bindings = getBindingsSync()
+  const parserOptions = getParserOptions(options)
+  const parsed = await bindings.parse(src, parserOptions)
+  return JSON.parse(parsed)
 }
 
 export function getBinaryMetadata() {
-  let bindings
-  try {
-    bindings = loadNative()
-  } catch (e) {
-    // Suppress exceptions, this fn allows to fail to load native bindings
-  }
-
   return {
-    target: bindings?.getTargetTriple?.(),
+    target: loadedBindings?.getTargetTriple?.(),
   }
 }
 
@@ -1597,8 +1594,8 @@ export function getBinaryMetadata() {
 export function initCustomTraceSubscriber(traceFileName?: string) {
   if (!swcTraceFlushGuard) {
     // Wasm binary doesn't support trace emission
-    let bindings = loadNative()
-    swcTraceFlushGuard = bindings.initCustomTraceSubscriber?.(traceFileName)
+    swcTraceFlushGuard =
+      getBindingsSync().initCustomTraceSubscriber?.(traceFileName)
   }
 }
 
@@ -1625,9 +1622,8 @@ function once(fn: () => void): () => void {
  */
 export const teardownTraceSubscriber = once(() => {
   try {
-    let bindings = loadNative()
     if (swcTraceFlushGuard) {
-      bindings.teardownTraceSubscriber?.(swcTraceFlushGuard)
+      getBindingsSync().teardownTraceSubscriber?.(swcTraceFlushGuard)
     }
   } catch (e) {
     // Suppress exceptions, this fn allows to fail to load native bindings
@@ -1637,14 +1633,12 @@ export const teardownTraceSubscriber = once(() => {
 export async function getModuleNamedExports(
   resourcePath: string
 ): Promise<string[]> {
-  const bindings = await loadBindings()
-  return bindings.rspack.getModuleNamedExports(resourcePath)
+  return getBindingsSync().rspack.getModuleNamedExports(resourcePath)
 }
 
 export async function warnForEdgeRuntime(
   source: string,
   isProduction: boolean
 ): Promise<NapiSourceDiagnostic[]> {
-  const bindings = await loadBindings()
-  return bindings.rspack.warnForEdgeRuntime(source, isProduction)
+  return getBindingsSync().rspack.warnForEdgeRuntime(source, isProduction)
 }