module: implement register utility

PR-URL: #46826
Reviewed-By: Jacob Smith <[email protected]>
Reviewed-By: Geoffrey Booth <[email protected]>
Reviewed-By: Antoine du Hamel <[email protected]>

Author: jlenon7

doc/api/errors.md

@@ -1233,6 +1233,23 @@ provided.
 Encoding provided to `TextDecoder()` API was not one of the
 [WHATWG Supported Encodings][].
 
+<a id="ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE"></a>
+
+### `ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Programmatically registering custom ESM loaders
+currently requires at least one custom loader to have been
+registered via the `--experimental-loader` flag. A no-op
+loader registered via CLI is sufficient
+(for example: `--experimental-loader data:text/javascript,`;
+do not omit the necessary trailing comma).
+A future version of Node.js will support the programmatic
+registration of loaders without needing to also use the flag.
+
 <a id="ERR_EVAL_ESM_CANNOT_PRINT"></a>
 
 ### `ERR_EVAL_ESM_CANNOT_PRINT`

doc/api/esm.md

@@ -1225,6 +1225,17 @@ console.log('some module!');
 If you run `node --experimental-loader ./import-map-loader.js main.js`
 the output will be `some module!`.
 
+### Register loaders programmatically
+
+<!-- YAML
+added: REPLACEME
+-->
+
+In addition to using the `--experimental-loader` option in the CLI,
+loaders can also be registered programmatically. You can find
+detailed information about this process in the documentation page
+for [`module.register()`][].
+
 ## Resolution and loading algorithm
 
 ### Features
@@ -1599,6 +1610,7 @@ for ESM specifiers is [commonjs-extension-resolution-loader][].
 [`import.meta.url`]: #importmetaurl
 [`import`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import
 [`module.createRequire()`]: module.md#modulecreaterequirefilename
+[`module.register()`]: module.md#moduleregister
 [`module.syncBuiltinESMExports()`]: module.md#modulesyncbuiltinesmexports
 [`package.json`]: packages.md#nodejs-packagejson-field-definitions
 [`port.ref()`]: https://nodejs.org/dist/latest-v17.x/docs/api/worker_threads.html#portref

doc/api/module.md

@@ -80,6 +80,101 @@ isBuiltin('fs'); // true
 isBuiltin('wss'); // false
 ```
 
+### `module.register()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+In addition to using the `--experimental-loader` option in the CLI,
+loaders can be registered programmatically using the
+`module.register()` method.
+
+```mjs
+import { register } from 'node:module';
+
+register('http-to-https', import.meta.url);
+
+// Because this is a dynamic `import()`, the `http-to-https` hooks will run
+// before importing `./my-app.mjs`.
+await import('./my-app.mjs');
+```
+
+In the example above, we are registering the `http-to-https` loader,
+but it will only be available for subsequently imported modules—in
+this case, `my-app.mjs`. If the `await import('./my-app.mjs')` had
+instead been a static `import './my-app.mjs'`, _the app would already
+have been loaded_ before the `http-to-https` hooks were
+registered. This is part of the design of ES modules, where static
+imports are evaluated from the leaves of the tree first back to the
+trunk. There can be static imports _within_ `my-app.mjs`, which
+will not be evaluated until `my-app.mjs` is when it's dynamically
+imported.
+
+The `--experimental-loader` flag of the CLI can be used together
+with the `register` function; the loaders registered with the
+function will follow the same evaluation chain of loaders registered
+within the CLI:
+
+```console
+node \
+  --experimental-loader unpkg \
+  --experimental-loader http-to-https \
+  --experimental-loader cache-buster \
+  entrypoint.mjs
+```
+
+```mjs
+// entrypoint.mjs
+import { URL } from 'node:url';
+import { register } from 'node:module';
+
+const loaderURL = new URL('./my-programmatically-loader.mjs', import.meta.url);
+
+register(loaderURL);
+await import('./my-app.mjs');
+```
+
+The `my-programmatic-loader.mjs` can leverage `unpkg`,
+`http-to-https`, and `cache-buster` loaders.
+
+It's also possible to use `register` more than once:
+
+```mjs
+// entrypoint.mjs
+import { URL } from 'node:url';
+import { register } from 'node:module';
+
+register(new URL('./first-loader.mjs', import.meta.url));
+register('./second-loader.mjs', import.meta.url);
+await import('./my-app.mjs');
+```
+
+Both loaders (`first-loader.mjs` and `second-loader.mjs`) can use
+all the resources provided by the loaders registered in the CLI. But
+remember that they will only be available in the next imported
+module (`my-app.mjs`). The evaluation order of the hooks when
+importing `my-app.mjs` and consecutive modules in the example above
+will be:
+
+```console
+resolve: second-loader.mjs
+resolve: first-loader.mjs
+resolve: cache-buster
+resolve: http-to-https
+resolve: unpkg
+load: second-loader.mjs
+load: first-loader.mjs
+load: cache-buster
+load: http-to-https
+load: unpkg
+globalPreload: second-loader.mjs
+globalPreload: first-loader.mjs
+globalPreload: cache-buster
+globalPreload: http-to-https
+globalPreload: unpkg
+```
+
 ### `module.syncBuiltinESMExports()`
 
 <!-- YAML

lib/internal/errors.js

@@ -1039,6 +1039,11 @@ E('ERR_ENCODING_INVALID_ENCODED_DATA', function(encoding, ret) {
 }, TypeError);
 E('ERR_ENCODING_NOT_SUPPORTED', 'The "%s" encoding is not supported',
   RangeError);
+E('ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE', 'Programmatically registering custom ESM loaders ' +
+  'currently requires at least one custom loader to have been registered via the --experimental-loader ' +
+  'flag. A no-op loader registered via CLI is sufficient (for example: `--experimental-loader ' +
+  '"data:text/javascript,"` with the necessary trailing comma). A future version of Node.js ' +
+  'will remove this requirement.', Error);
 E('ERR_EVAL_ESM_CANNOT_PRINT', '--print cannot be used with ESM input', Error);
 E('ERR_EVENT_RECURSION', 'The event "%s" is already being dispatched', Error);
 E('ERR_FALSY_VALUE_REJECTION', function(reason) {

lib/internal/modules/esm/hooks.js

@@ -45,6 +45,8 @@ const {
   validateString,
 } = require('internal/validators');
 
+const { kEmptyObject } = require('internal/util');
+
 const {
   defaultResolve,
   throwIfInvalidParentURL,
@@ -117,6 +119,23 @@ class Hooks {
   // Cache URLs we've already validated to avoid repeated validation
   #validatedUrls = new SafeSet();
 
+  /**
+   * Import and register custom/user-defined module loader hook(s).
+   * @param {string} urlOrSpecifier
+   * @param {string} parentURL
+   */
+  async register(urlOrSpecifier, parentURL) {
+    const moduleLoader = require('internal/process/esm_loader').esmLoader;
+
+    const keyedExports = await moduleLoader.import(
+      urlOrSpecifier,
+      parentURL,
+      kEmptyObject,
+    );
+
+    this.addCustomLoader(urlOrSpecifier, keyedExports);
+  }
+
   /**
    * Collect custom/user-defined module loader hook(s).
    * After all hooks have been collected, the global preload hook(s) must be initialized.

lib/internal/modules/esm/loader.js

@@ -11,6 +11,7 @@ const {
 } = primordials;
 
 const {
+  ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE,
   ERR_UNKNOWN_MODULE_FORMAT,
 } = require('internal/errors').codes;
 const { getOptionValue } = require('internal/options');
@@ -287,12 +288,19 @@ class CustomizedModuleLoader extends DefaultModuleLoader {
   constructor() {
     super();
 
-    if (hooksProxy) {
-      // The worker proxy is shared across all instances, so don't recreate it if it already exists.
-      return;
-    }
-    const { HooksProxy } = require('internal/modules/esm/hooks');
-    hooksProxy = new HooksProxy(); // The user's custom hooks are loaded within the worker as part of its startup.
+    getHooksProxy();
+  }
+
+  /**
+   * Register some loader specifier.
+   * @param {string} originalSpecifier The specified URL path of the loader to
+   *                                   be registered.
+   * @param {string} parentURL The parent URL from where the loader will be
+   *                           registered if using it package name as specifier
+   * @returns {{ format: string, url: URL['href'] }}
+   */
+  register(originalSpecifier, parentURL) {
+    return hooksProxy.makeSyncRequest('register', originalSpecifier, parentURL);
   }
 
   /**
@@ -370,7 +378,46 @@ function createModuleLoader(useCustomLoadersIfPresent = true) {
   return new DefaultModuleLoader();
 }
 
+/**
+ * Get the HooksProxy instance. If it is not defined, then create a new one.
+ * @returns {HooksProxy}
+ */
+function getHooksProxy() {
+  if (!hooksProxy) {
+    const { HooksProxy } = require('internal/modules/esm/hooks');
+    hooksProxy = new HooksProxy();
+  }
+
+  return hooksProxy;
+}
+
+/**
+ * Register a single loader programmatically.
+ * @param {string} specifier
+ * @param {string} [parentURL]
+ * @returns {void}
+ * @example
+ * ```js
+ * register('./myLoader.js');
+ * register('ts-node/esm', import.meta.url);
+ * register('./myLoader.js', import.meta.url);
+ * register(new URL('./myLoader.js', import.meta.url));
+ * ```
+ */
+function register(specifier, parentURL = 'data:') {
+  // TODO: Remove this limitation in a follow-up before `register` is released publicly
+  if (getOptionValue('--experimental-loader').length < 1) {
+    throw new ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE();
+  }
+
+  const moduleLoader = require('internal/process/esm_loader').esmLoader;
+
+  moduleLoader.register(`${specifier}`, parentURL);
+}
+
 module.exports = {
   DefaultModuleLoader,
   createModuleLoader,
+  getHooksProxy,
+  register,
 };

lib/internal/modules/esm/utils.js

@@ -146,9 +146,9 @@ async function initializeHooks() {
     load(url, context) { return hooks.load(url, context); }
   }
   const privateModuleLoader = new ModuleLoader();
-
   const parentURL = pathToFileURL(cwd).href;
 
+  // TODO(jlenon7): reuse the `Hooks.register()` method for registering loaders.
   for (let i = 0; i < customLoaderURLs.length; i++) {
     const customLoaderURL = customLoaderURLs[i];
 

lib/module.js

@@ -2,8 +2,10 @@
 
 const { findSourceMap } = require('internal/source_map/source_map_cache');
 const { Module } = require('internal/modules/cjs/loader');
+const { register } = require('internal/modules/esm/loader');
 const { SourceMap } = require('internal/source_map/source_map');
 
 Module.findSourceMap = findSourceMap;
+Module.register = register;
 Module.SourceMap = SourceMap;
 module.exports = Module;

test/es-module/test-esm-loader-hooks.mjs

@@ -24,6 +24,28 @@ describe('Loader hooks', { concurrency: true }, () => {
     assert.match(lines[3], /{"source":{"type":"Buffer","data":\[.*\]},"format":"json","shortCircuit":true}/);
   });
 
+  it('are called with all expected arguments using register function', async () => {
+    const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [
+      '--no-warnings',
+      '--experimental-loader=data:text/javascript,',
+      '--input-type=module',
+      '--eval',
+      "import { register } from 'node:module';" +
+      `register(${JSON.stringify(fixtures.fileURL('/es-module-loaders/hooks-input.mjs'))});` +
+      `await import(${JSON.stringify(fixtures.fileURL('/es-modules/json-modules.mjs'))});`,
+    ]);
+
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(code, 0);
+    assert.strictEqual(signal, null);
+
+    const lines = stdout.split('\n');
+    assert.match(lines[0], /{"url":"file:\/\/\/.*\/json-modules\.mjs","format":"test","shortCircuit":true}/);
+    assert.match(lines[1], /{"source":{"type":"Buffer","data":\[.*\]},"format":"module","shortCircuit":true}/);
+    assert.match(lines[2], /{"url":"file:\/\/\/.*\/experimental\.json","format":"test","shortCircuit":true}/);
+    assert.match(lines[3], /{"source":{"type":"Buffer","data":\[.*\]},"format":"json","shortCircuit":true}/);
+  });
+
   describe('should handle never-settling hooks in ESM files', { concurrency: true }, () => {
     it('top-level await of a never-settling resolve', async () => {
       const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [