esm: loader chaining

This patch adds the ability to chain loaders together. Hooks still need
auditing on the best way to behave in the context of chaining, and that
will be addressed in future patches.

Author: devsnek

doc/api/esm.md

@@ -9,7 +9,8 @@ const {
 
 const esmLoader = require('internal/process/esm_loader');
 const {
-  evalScript
+  evalScript,
+  uncaughtException,
 } = require('internal/process/execution');
 
 const console = require('internal/console/global');
@@ -33,7 +34,7 @@ if (process.env.NODE_REPL_EXTERNAL_MODULE) {
     process.exit(1);
   }
 
-  esmLoader.loadESM(() => {
+  esmLoader.getLoader().then(() => {
     console.log(`Welcome to Node.js ${process.version}.\n` +
       'Type ".help" for more information.');
 
@@ -61,5 +62,7 @@ if (process.env.NODE_REPL_EXTERNAL_MODULE) {
                  getOptionValue('--inspect-brk'),
                  getOptionValue('--print'));
     }
+  }).catch((e) => {
+    uncaughtException(e, true /* fromPromise */);
   });
 }

lib/internal/main/repl.js

@@ -102,6 +102,13 @@ const {
 const { validateString } = require('internal/validators');
 const pendingDeprecation = getOptionValue('--pending-deprecation');
 
+const originalModuleExports = new SafeWeakMap();
+module.exports = {
+  wrapSafe, Module, toRealPath, readPackageScope,
+  originalModuleExports,
+  get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; }
+};
+
 const {
   CHAR_FORWARD_SLASH,
   CHAR_BACKWARD_SLASH,
@@ -113,8 +120,6 @@ const {
 } = require('internal/util/types');
 
 const asyncESM = require('internal/process/esm_loader');
-const ModuleJob = require('internal/modules/esm/module_job');
-const { ModuleWrap, kInstantiated } = internalBinding('module_wrap');
 const {
   encodedSepRegEx,
   packageInternalResolve
@@ -1119,30 +1124,7 @@ Module.prototype.load = function(filename) {
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
-  const ESMLoader = asyncESM.ESMLoader;
-  const url = `${pathToFileURL(filename)}`;
-  const module = ESMLoader.moduleMap.get(url);
-  // Create module entry at load time to snapshot exports correctly
-  const exports = this.exports;
-  // Called from cjs translator
-  if (module !== undefined && module.module !== undefined) {
-    if (module.module.getStatus() >= kInstantiated)
-      module.module.setExport('default', exports);
-  } else {
-    // Preemptively cache
-    // We use a function to defer promise creation for async hooks.
-    ESMLoader.moduleMap.set(
-      url,
-      // Module job creation will start promises.
-      // We make it a function to lazily trigger those promises
-      // for async hooks compatibility.
-      () => new ModuleJob(ESMLoader, url, () =>
-        new ModuleWrap(url, undefined, ['default'], function() {
-          this.setExport('default', exports);
-        })
-      , false /* isMain */, false /* inspectBrk */)
-    );
-  }
+  originalModuleExports.set(this, this.exports);
 };
 
 
@@ -1176,7 +1158,7 @@ function wrapSafe(filename, content, cjsModuleInstance) {
       lineOffset: 0,
       displayErrors: true,
       importModuleDynamically: async (specifier) => {
-        const loader = asyncESM.ESMLoader;
+        const loader = await asyncESM.getLoader();
         return loader.import(specifier, normalizeReferrerURL(filename));
       },
     });
@@ -1209,7 +1191,7 @@ function wrapSafe(filename, content, cjsModuleInstance) {
   const { callbackMap } = internalBinding('module_wrap');
   callbackMap.set(compiled.cacheKey, {
     importModuleDynamically: async (specifier) => {
-      const loader = asyncESM.ESMLoader;
+      const loader = await asyncESM.getLoader();
       return loader.import(specifier, normalizeReferrerURL(filename));
     }
   });

lib/internal/modules/cjs/loader.js

@@ -33,7 +33,7 @@ if (experimentalWasmModules)
 if (experimentalJsonModules)
   extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json';
 
-function defaultGetFormat(url, context, defaultGetFormatUnused) {
+function defaultGetFormat(url, context, nextGetFormat) {
   if (StringPrototypeStartsWith(url, 'nodejs:')) {
     return { format: 'builtin' };
   }

lib/internal/modules/esm/get_format.js

@@ -6,15 +6,14 @@ require('internal/modules/cjs/loader');
 const {
   FunctionPrototypeBind,
   ObjectSetPrototypeOf,
-  SafeMap,
 } = primordials;
 
 const {
   ERR_INVALID_ARG_VALUE,
   ERR_INVALID_RETURN_PROPERTY,
   ERR_INVALID_RETURN_PROPERTY_VALUE,
   ERR_INVALID_RETURN_VALUE,
-  ERR_UNKNOWN_MODULE_FORMAT
+  ERR_UNKNOWN_MODULE_FORMAT,
 } = require('internal/errors').codes;
 const { URL, pathToFileURL } = require('internal/url');
 const { validateString } = require('internal/validators');
@@ -28,11 +27,30 @@ const {
 const { defaultGetFormat } = require('internal/modules/esm/get_format');
 const { defaultGetSource } = require(
   'internal/modules/esm/get_source');
-const { defaultTransformSource } = require(
-  'internal/modules/esm/transform_source');
 const { translators } = require(
   'internal/modules/esm/translators');
 const { getOptionValue } = require('internal/options');
+const {
+  isArrayBufferView,
+  isAnyArrayBuffer,
+} = require('internal/util/types');
+
+let cwd; // Initialized in importLoader
+
+function validateSource(source, hookName, allowString) {
+  if (allowString && typeof source === 'string') {
+    return;
+  }
+  if (isArrayBufferView(source) || isAnyArrayBuffer(source)) {
+    return;
+  }
+  throw new ERR_INVALID_RETURN_PROPERTY_VALUE(
+    `${allowString ? 'string, ' : ''}array buffer, or typed array`,
+    hookName,
+    'source',
+    source,
+  );
+}
 
 /* A Loader instance is used as the main entry point for loading ES modules.
  * Currently, this is a singleton -- there is only one used for loading
@@ -46,33 +64,16 @@ class Loader {
     // Registry of loaded modules, akin to `require.cache`
     this.moduleMap = new ModuleMap();
 
-    // Map of already-loaded CJS modules to use
-    this.cjsCache = new SafeMap();
-
-    // This hook is called before the first root module is imported. It's a
-    // function that returns a piece of code that runs as a sloppy-mode script.
-    // The script may evaluate to a function that can be called with a
-    // `getBuiltin` helper that can be used to retrieve builtins.
-    // If the hook returns `null` instead of a source string, it opts out of
-    // running any preload code.
-    // The preload code runs as soon as the hook module has finished evaluating.
-    this._getGlobalPreloadCode = null;
-    // The resolver has the signature
-    //   (specifier : string, parentURL : string, defaultResolve)
-    //       -> Promise<{ url : string }>
-    // where defaultResolve is ModuleRequest.resolve (having the same
-    // signature itself).
+    // Preload code is provided by loaders to be run after hook initialization.
+    this.globalPreloadCode = [];
+    // Loader resolve hook.
     this._resolve = defaultResolve;
-    // This hook is called after the module is resolved but before a translator
-    // is chosen to load it; the format returned by this function is the name
-    // of a translator.
+    // Loader getFormat hook.
     this._getFormat = defaultGetFormat;
-    // This hook is called just before the source code of an ES module file
-    // is loaded.
+    // Loader getSource hook.
     this._getSource = defaultGetSource;
-    // This hook is called just after the source code of an ES module file
-    // is loaded, but before anything is done with the string.
-    this._transformSource = defaultTransformSource;
+    // Transform source hooks.
+    this.transformSourceHooks = [];
     // The index for assigning unique URLs to anonymous module evaluation
     this.evalIndex = 0;
   }
@@ -83,7 +84,7 @@ class Loader {
       validateString(parentURL, 'parentURL');
 
     const resolveResponse = await this._resolve(
-      specifier, { parentURL, conditions: DEFAULT_CONDITIONS }, defaultResolve);
+      specifier, { parentURL, conditions: DEFAULT_CONDITIONS });
     if (typeof resolveResponse !== 'object') {
       throw new ERR_INVALID_RETURN_VALUE(
         'object', 'loader resolve', resolveResponse);
@@ -98,8 +99,7 @@ class Loader {
   }
 
   async getFormat(url) {
-    const getFormatResponse = await this._getFormat(
-      url, {}, defaultGetFormat);
+    const getFormatResponse = await this._getFormat(url, {});
     if (typeof getFormatResponse !== 'object') {
       throw new ERR_INVALID_RETURN_VALUE(
         'object', 'loader getFormat', getFormatResponse);
@@ -137,6 +137,22 @@ class Loader {
     return format;
   }
 
+  async getSource(url, format) {
+    const { source: originalSource } = await this._getSource(url, { format });
+
+    const allowString = format !== 'wasm';
+    validateSource(originalSource, 'getSource', allowString);
+
+    let source = originalSource;
+    for (let i = 0; i < this.transformSourceHooks.length; i += 1) {
+      const hook = this.transformSourceHooks[i];
+      ({ source } = await hook(source, { url, format, originalSource }));
+      validateSource(source, 'transformSource', allowString);
+    }
+
+    return source;
+  }
+
   async eval(
     source,
     url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href
@@ -166,72 +182,84 @@ class Loader {
     return module.getNamespace();
   }
 
+  async importLoader(specifier) {
+    if (cwd === undefined) {
+      try {
+        // `process.cwd()` can fail.
+        cwd = process.cwd() + '/';
+      } catch {
+        cwd = 'file:///';
+      }
+      cwd = pathToFileURL(cwd).href;
+    }
+
+    const { url } = await defaultResolve(specifier, cwd,
+                                         { conditions: DEFAULT_CONDITIONS });
+    const { format } = await defaultGetFormat(url, {});
+
+    // !!! CRITICAL SECTION !!!
+    // NO AWAIT OPS BETWEEN HERE AND SETTING JOB IN MODULE MAP!
+    // YIELDING CONTROL COULD RESULT IN MAP BEING OVERRIDDEN!
+    let job = this.moduleMap.get(url);
+    if (job === undefined) {
+      if (!translators.has(format))
+        throw new ERR_UNKNOWN_MODULE_FORMAT(format);
+
+      const loaderInstance = translators.get(format);
+
+      job = new ModuleJob(this, url, loaderInstance, false, false);
+      this.moduleMap.set(url, job);
+      // !!! END CRITICAL SECTION !!!
+    }
+
+    const { module } = await job.run();
+    return module.getNamespace();
+  }
+
   hook(hooks) {
     const {
       resolve,
-      dynamicInstantiate,
       getFormat,
       getSource,
-      transformSource,
-      getGlobalPreloadCode,
     } = hooks;
 
     // Use .bind() to avoid giving access to the Loader instance when called.
     if (resolve !== undefined)
       this._resolve = FunctionPrototypeBind(resolve, null);
-    if (dynamicInstantiate !== undefined) {
-      process.emitWarning(
-        'The dynamicInstantiate loader hook has been removed.');
-    }
     if (getFormat !== undefined) {
       this._getFormat = FunctionPrototypeBind(getFormat, null);
     }
     if (getSource !== undefined) {
       this._getSource = FunctionPrototypeBind(getSource, null);
     }
-    if (transformSource !== undefined) {
-      this._transformSource = FunctionPrototypeBind(transformSource, null);
-    }
-    if (getGlobalPreloadCode !== undefined) {
-      this._getGlobalPreloadCode =
-        FunctionPrototypeBind(getGlobalPreloadCode, null);
-    }
   }
 
   runGlobalPreloadCode() {
-    if (!this._getGlobalPreloadCode) {
-      return;
-    }
-    const preloadCode = this._getGlobalPreloadCode();
-    if (preloadCode === null) {
-      return;
-    }
+    for (let i = 0; i < this.globalPreloadCode.length; i += 1) {
+      const preloadCode = this.globalPreloadCode[i];
 
-    if (typeof preloadCode !== 'string') {
-      throw new ERR_INVALID_RETURN_VALUE(
-        'string', 'loader getGlobalPreloadCode', preloadCode);
+      const { compileFunction } = require('vm');
+      const preloadInit = compileFunction(preloadCode, ['getBuiltin'], {
+        filename: '<preload>',
+      });
+      const { NativeModule } = require('internal/bootstrap/loaders');
+
+      preloadInit.call(globalThis, (builtinName) => {
+        if (NativeModule.canBeRequiredByUsers(builtinName)) {
+          return require(builtinName);
+        }
+        throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName);
+      });
     }
-    const { compileFunction } = require('vm');
-    const preloadInit = compileFunction(preloadCode, ['getBuiltin'], {
-      filename: '<preload>',
-    });
-    const { NativeModule } = require('internal/bootstrap/loaders');
-
-    preloadInit.call(globalThis, (builtinName) => {
-      if (NativeModule.canBeRequiredByUsers(builtinName)) {
-        return require(builtinName);
-      }
-      throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName);
-    });
   }
 
   async getModuleJob(specifier, parentURL) {
     const url = await this.resolve(specifier, parentURL);
     const format = await this.getFormat(url);
+
+    // !!! CRITICAL SECTION !!!
+    // NO AWAIT OPS BETWEEN HERE AND SETTING JOB IN MODULE MAP
     let job = this.moduleMap.get(url);
-    // CommonJS will set functions for lazy job evaluation.
-    if (typeof job === 'function')
-      this.moduleMap.set(url, job = job());
     if (job !== undefined)
       return job;
 
@@ -245,6 +273,8 @@ class Loader {
     job = new ModuleJob(this, url, loaderInstance, parentURL === undefined,
                         inspectBrk);
     this.moduleMap.set(url, job);
+    // !!! END CRITICAL SECTION !!!
+
     return job;
   }
 }

lib/internal/modules/esm/loader.js

@@ -764,8 +764,7 @@ function resolveAsCommonJS(specifier, parentURL) {
   }
 }
 
-function defaultResolve(specifier, context = {}, defaultResolveUnused) {
-  let { parentURL, conditions } = context;
+function defaultResolve(specifier, { parentURL, conditions } = {}) {
   let parsed;
   try {
     parsed = new URL(specifier);