feat(compartment-mapper): expose AST-based CJS parser

This creates a `parse-cjs-babel` parser which uses `CjsModuleSource` from `@endo/module-source` instead of the lexer, supporting `import()` in CJS sources.

Exposed within `parserForLanguageWithCjsBabel` as exported from `import-parsers.js`.

Refactored compat tests to run across both parsers.

Author: boneskull

.changeset/olive-bats-push.md

@@ -0,0 +1,7 @@
+---
+'@endo/compartment-mapper': minor
+'@endo/module-source': minor
+'@endo/parser-pipeline': patch
+---
+
+Exposes a Babel-based AST parser for CJS in `@endo/compartment-mapper` which supports dynamic `import()`. `@endo/module-source` exposes `CjsModuleSource` and `createCjsModuleSourcePasses()` (for use with `@endo/parser-pipeline`) and contains the implementation of the Babel-based parser for CJS. `@endo/parser-pipeline` now accepts records created by `CjsModuleSource`.

packages/compartment-mapper/cjs.js

@@ -0,0 +1 @@
+export * from './src/cjs.js';

packages/compartment-mapper/import-parsers.js

@@ -1,4 +1,7 @@
 // eslint-disable-next-line import/export -- just types
 export * from './src/types-external.js';
 
-export { defaultParserForLanguage } from './src/import-parsers.js';
+export {
+  defaultParserForLanguage,
+  parserForLanguageWithCjsBabel,
+} from './src/import-parsers.js';

packages/compartment-mapper/package.json

@@ -23,6 +23,7 @@
   "main": "./index.js",
   "exports": {
     ".": "./index.js",
+    "./cjs.js": "./cjs.js",
     "./import.js": "./import.js",
     "./import-lite.js": "./import-lite.js",
     "./import-parsers.js": "./import-parsers.js",

packages/compartment-mapper/src/cjs.js

@@ -0,0 +1,79 @@
+/* eslint-disable no-underscore-dangle */
+/**
+ * Provides {@link buildCjsExecuteRecord}, a function that converts a
+ * {@link CjsModuleSourceRecord} into a {@link FinalStaticModuleType}.
+ *
+ * For use with `@endo/parser-pipeline`'s `createComposedParser()`.
+ *
+ * @module
+ */
+
+import { getModulePaths, wrap } from './parse-cjs-shared-export-wrapper.js';
+
+/**
+ * @import {CjsModuleSourceRecord} from '@endo/module-source'
+ * @import {ReadFn, ReadPowers} from './types.js'
+ * @import {FinalStaticModuleType} from 'ses'
+ */
+
+const { freeze } = Object;
+
+/**
+ * Converts a {@link CjsModuleSourceRecord} (which has a `cjsFunctor` string)
+ * into a `FinalStaticModuleType`-compatible record (which has an `execute`
+ * function). This is the bridge between the composed-pipeline CJS analysis and
+ * the compartment-mapper execution model.
+ *
+ * Used by both {@link parseCjsBabel} (single-shot parser) and
+ * {@link createCjsExecParser} (composed-pipeline parser) so the execution
+ * logic lives in exactly one place.
+ *
+ * @param {CjsModuleSourceRecord} cjsRecord
+ * @param {string} location
+ * @param {ReadFn | ReadPowers | undefined} readPowers
+ * @returns {FinalStaticModuleType}
+ */
+
+export const buildCjsExecuteRecord = (cjsRecord, location, readPowers) => {
+  const { filename, dirname } = getModulePaths(readPowers, location);
+
+  /**
+   * @param {object} moduleEnvironmentRecord
+   * @param {Compartment} compartment
+   * @param {Record<string, string>} resolvedImports
+   */
+  const execute = (moduleEnvironmentRecord, compartment, resolvedImports) => {
+    const functor = compartment.evaluate(cjsRecord.cjsFunctor);
+
+    const wrapResult = wrap({
+      moduleEnvironmentRecord,
+      compartment,
+      resolvedImports,
+      location,
+      readPowers,
+    });
+
+    const args = [
+      wrapResult.require,
+      wrapResult.moduleExports,
+      wrapResult.module,
+      filename,
+      dirname,
+    ];
+
+    if (cjsRecord.__needsImport__ && wrapResult.importFn) {
+      args.push(wrapResult.importFn);
+    }
+
+    functor.call(wrapResult.moduleExports, ...args);
+
+    wrapResult.afterExecute();
+  };
+
+  return freeze({
+    imports: cjsRecord.imports,
+    exports: cjsRecord.exports,
+    reexports: cjsRecord.reexports,
+    execute,
+  });
+};

packages/compartment-mapper/src/import-parsers.js

@@ -11,9 +11,12 @@ import parserText from './parse-text.js';
 import parserBytes from './parse-bytes.js';
 import parserCjs from './parse-cjs.js';
 import parserMjs from './parse-mjs.js';
+import parserCjsBabel from './parse-cjs-babel.js';
+
+const { freeze } = Object;
 
 /** @satisfies {Readonly<ParserForLanguage>} */
-export const defaultParserForLanguage = Object.freeze(
+export const defaultParserForLanguage = freeze(
   /** @type {const} */ ({
     mjs: parserMjs,
     cjs: parserCjs,
@@ -22,3 +25,11 @@ export const defaultParserForLanguage = Object.freeze(
     bytes: parserBytes,
   }),
 );
+
+/** @satisfies {Readonly<ParserForLanguage>} */
+export const parserForLanguageWithCjsBabel = freeze(
+  /** @type {const} */ ({
+    ...defaultParserForLanguage,
+    cjs: parserCjsBabel,
+  }),
+);

packages/compartment-mapper/src/parse-cjs-babel.js

@@ -0,0 +1,52 @@
+/* eslint-disable no-underscore-dangle */
+/**
+ * Provides language behavior (parser) for importing CommonJS as a virtual
+ * module source, using Babel AST analysis instead of the character-level lexer.
+ *
+ * Drop-in replacement for {@link parse-cjs.js}. Consumers opt in via the
+ * pre-built parser map:
+ *
+ * ```js
+ * import { parserForLanguageWithCjsBabel } from '@endo/compartment-mapper/import-parsers.js';
+ *
+ * await importLocation(readPowers, entryUrl, {
+ *   parserForLanguage: parserForLanguageWithCjsBabel,
+ * });
+ * ```
+ *
+ * @module
+ */
+
+/**
+ * @import {ParseFn, ParserImplementation} from './types.js'
+ */
+
+import { CjsModuleSource } from '@endo/module-source';
+import { buildCjsExecuteRecord } from './cjs.js';
+
+const textDecoder = new TextDecoder();
+
+/** @type {ParseFn} */
+export const parseCjsBabel = (
+  bytes,
+  _specifier,
+  location,
+  _packageLocation,
+  { readPowers } = {},
+) => {
+  const source = textDecoder.decode(bytes);
+  const cjsRecord = new CjsModuleSource(source, { sourceUrl: location });
+
+  return {
+    parser: 'cjs',
+    bytes,
+    record: buildCjsExecuteRecord(cjsRecord, location, readPowers),
+  };
+};
+
+/** @type {ParserImplementation} */
+export default {
+  parse: parseCjsBabel,
+  heuristicImports: true,
+  synchronous: true,
+};

packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js

@@ -83,10 +83,11 @@ export const getModulePaths = (readPowers, location) => {
  * @param {string} in.location
  * @param {ReadFn | ReadPowers | undefined} in.readPowers
  * @returns {{
- *   module: { exports: any },
- *   moduleExports: any,
- *   afterExecute: Function,
- *   require: Function,
+ *   module: { exports: unknown },
+ *   moduleExports: unknown,
+ *   afterExecute: () => void,
+ *   require: (specifier: string) => unknown,
+ *   importFn: (specifier: string) => Promise<unknown>,
  * }}
  */
 export const wrap = ({
@@ -204,6 +205,15 @@ export const wrap = ({
 
   freeze(require);
 
+  /** @param {string} importSpecifier */
+  const importFn = async importSpecifier => {
+    const specifier = has(resolvedImports, importSpecifier)
+      ? resolvedImports[importSpecifier]
+      : importSpecifier;
+    return compartment.import(specifier);
+  };
+  freeze(importFn);
+
   const afterExecute = () => {
     const finalExports = module.exports; // in case it's a getter, only call it once
     const exportsHaveBeenOverwritten = finalExports !== originalExports;
@@ -231,5 +241,6 @@ export const wrap = ({
     moduleExports: originalExports,
     afterExecute,
     require,
+    importFn,
   };
 };

packages/compartment-mapper/test/cjs-compat.test.js

@@ -6,9 +6,14 @@ import 'ses';
 import test from 'ava';
 import path from 'path';
 import { scaffold } from './scaffold.js';
+import {
+  defaultParserForLanguage,
+  parserForLanguageWithCjsBabel,
+} from '../src/import-parsers.js';
 
 /**
  * @import {FixtureAssertionFn} from './test.types.js';
+ * @import {ThirdPartyStaticModuleInterface} from 'ses'
  */
 
 const fixture = new URL(
@@ -19,9 +24,13 @@ const fixtureDirname = new URL(
   'fixtures-cjs-compat/node_modules/app/dirname.js',
   import.meta.url,
 ).toString();
+const fixtureDynamicImport = new URL(
+  'fixtures-cjs-compat/node_modules/dynamic-import/index.js',
+  import.meta.url,
+).toString();
 
 const q = JSON.stringify;
-
+const { freeze } = Object;
 /**
  * @type {FixtureAssertionFn<{requireResolvePaths: string[]}>}
  */
@@ -70,50 +79,94 @@ const assertFixture = (t, { namespace, testCategoryHint }) => {
 
 const fixtureAssertionCount = 2;
 
-scaffold(
-  'fixtures-cjs-compat',
-  test,
-  fixture,
-  assertFixture,
-  fixtureAssertionCount,
-);
+const parsersForLanguage = {
+  default: defaultParserForLanguage,
+  babel: parserForLanguageWithCjsBabel,
+};
+
+for (const [name, parserForLanguage] of Object.entries(parsersForLanguage)) {
+  scaffold(
+    `fixtures-cjs-compat-${name}`,
+    test,
+    fixture,
+    assertFixture,
+    fixtureAssertionCount,
+    {
+      parserForLanguage,
+    },
+  );
 
-// Exit module errors are also deferred
-scaffold(
-  'fixtures-cjs-compat-exit-module',
-  test,
-  fixture,
-  assertFixture,
-  fixtureAssertionCount,
-  {
-    additionalOptions: {
-      importHook: async specifier => {
-        throw Error(`${q(specifier)} is NOT an exit module.`);
+  // Exit module errors are also deferred
+  scaffold(
+    `fixtures-cjs-compat-exit-module-${name}`,
+    test,
+    fixture,
+    assertFixture,
+    fixtureAssertionCount,
+    {
+      additionalOptions: {
+        importHook: async specifier => {
+          throw Error(`${q(specifier)} is NOT an exit module.`);
+        },
       },
+      parserForLanguage,
     },
-  },
-);
+  );
 
-scaffold(
-  'fixtures-cjs-compat-__dirname',
-  test,
-  fixtureDirname,
-  (t, { namespace, testCategoryHint }) => {
-    if (testCategoryHint === 'Location') {
-      const { __filename, __dirname } = namespace;
-      t.is(__filename, path.join(__dirname, '/dirname.js'));
-      t.assert(!__dirname.startsWith('file://'));
-      t.notRegex(
-        __dirname,
-        /[\\/]$/,
-        'Expected __dirname to NOT have a trailing slash',
-      );
-    } else {
-      const { __filename, __dirname } = namespace;
-      t.is(__dirname, null);
-      t.is(__filename, null);
-      t.pass();
-    }
-  },
-  3,
-);
+  scaffold(
+    `fixtures-cjs-compat-__dirname-${name}`,
+    test,
+    fixtureDirname,
+    (t, { namespace, testCategoryHint }) => {
+      if (testCategoryHint === 'Location') {
+        const { __filename, __dirname } = namespace;
+        t.is(__filename, path.join(__dirname, '/dirname.js'));
+        t.assert(!__dirname.startsWith('file://'));
+        t.notRegex(
+          __dirname,
+          /[\\/]$/,
+          'Expected __dirname to NOT have a trailing slash',
+        );
+      } else {
+        const { __filename, __dirname } = namespace;
+        t.is(__dirname, null);
+        t.is(__filename, null);
+        t.pass();
+      }
+    },
+    3,
+    {
+      parserForLanguage,
+    },
+  );
+
+  scaffold(
+    `fixtures-cjs-compat-dynamic-import-${name}`,
+    test,
+    fixtureDynamicImport,
+    async (t, { namespace }) => {
+      const { namespace: dynamicNamespace } =
+        // @ts-expect-error - untyped
+        await namespace.dynamicImport('a');
+      t.is(dynamicNamespace.foo, 'foo');
+    },
+    1,
+    {
+      // NOTE: this should fail with parse-cjs, but not parse-cjs-babel
+      knownFailure: name === 'default',
+      parserForLanguage,
+      additionalOptions: {
+        importHook: async () => {
+          /** @type {ThirdPartyStaticModuleInterface} */
+          return freeze({
+            imports: [],
+            exports: ['foo'],
+            execute: moduleExports => {
+              moduleExports.foo = 'foo';
+            },
+          });
+        },
+      },
+    },
+  );
+}