From f1c2775e4c2118f84bd2eb1001150f02c2d896db Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Wed, 15 Apr 2026 15:57:27 -0700 Subject: [PATCH 1/2] fix(module-source,compartment-mapper): fix SourceMapHook & other types This fixes the type of `SourceMapHook` and the implementation to align with reality. - Introduces some `.ts` sources for types; fixes `tsconfig.json` - Fixes exports of said types - Fixes shim types - Adds a `typedoc.json` for proper entry points - Adds type arguments for `ParserImplementation`, `ParseFn`, etc. for external parsers using custom policies. --- .../src/types/compartment-map-schema.ts | 40 ++++++++-- .../compartment-mapper/src/types/external.ts | 52 +++++++++---- .../src/types/policy-schema.ts | 78 ++++++++++++------- .../src/types/typescript.ts | 8 ++ .../module-source/src/external.types.d.ts | 1 + 5 files changed, 130 insertions(+), 49 deletions(-) diff --git a/packages/compartment-mapper/src/types/compartment-map-schema.ts b/packages/compartment-mapper/src/types/compartment-map-schema.ts index 0cc4b32392..1a14508bdf 100644 --- a/packages/compartment-mapper/src/types/compartment-map-schema.ts +++ b/packages/compartment-mapper/src/types/compartment-map-schema.ts @@ -14,7 +14,7 @@ import type { } from '../policy-format.js'; import type { CanonicalName } from './canonical-name.js'; import type { FileUrlString } from './external.js'; -import type { SomePackagePolicy } from './policy-schema.js'; +import type { PackagePolicy, SomePackagePolicy } from './policy-schema.js'; import type { PatternDescriptor } from './pattern-replacement.js'; import type { LiteralUnion } from './typescript.js'; @@ -122,23 +122,24 @@ export interface PackageCompartmentDescriptor * one for a given library or application `package.json`. */ export interface CompartmentDescriptor< - T extends ModuleConfiguration = ModuleConfiguration, - U extends string = string, + TModuleConfiguration extends ModuleConfiguration = ModuleConfiguration, + TCompartmentName extends string = string, + TPackagePolicy extends SomePackagePolicy = SomePackagePolicy, > { - label: CanonicalName; + label: CanonicalName; /** * the name of the originating package suitable for constructing a sourceURL * prefix that will match it to files in a developer workspace. */ name: string; - modules: Record; + modules: Record; scopes?: Record; /** language for extension */ parsers?: LanguageForExtension; /** language for module specifier */ types?: LanguageForModuleSpecifier; /** policy specific to compartment */ - policy?: SomePackagePolicy; + policy?: TPackagePolicy; location: string; /** @@ -154,9 +155,32 @@ export interface CompartmentDescriptor< retained?: true; } +/** + * Any {@link CompartmentDescriptor} + */ +export type SomeCompartmentDescriptor = CompartmentDescriptor; + +/** + * Any {@link CompartmentDescriptor} with a non-nullish + * {@link CompartmentDescriptor.policy} property + */ +export type SomeCompartmentDescriptorWithPolicy = + CompartmentDescriptorWithPolicy; + +/** + * A {@link CompartmentDescriptor} with a non-nullish + * {@link CompartmentDescriptor.policy} property + */ export type CompartmentDescriptorWithPolicy< - T extends ModuleConfiguration = ModuleConfiguration, -> = Omit, 'policy'> & { policy: SomePackagePolicy }; + TModuleConfiguration extends ModuleConfiguration = ModuleConfiguration, + TCompartmentName extends string = string, + TPackagePolicy extends SomePackagePolicy = SomePackagePolicy, +> = Omit< + CompartmentDescriptor, + 'policy' +> & { + policy: TPackagePolicy; +}; /** * A compartment descriptor digested by `digestCompartmentMap()` diff --git a/packages/compartment-mapper/src/types/external.ts b/packages/compartment-mapper/src/types/external.ts index 1d71c48177..a75621a593 100644 --- a/packages/compartment-mapper/src/types/external.ts +++ b/packages/compartment-mapper/src/types/external.ts @@ -19,6 +19,7 @@ import type { } from '../policy-format.js'; import type { CanonicalName } from './canonical-name.js'; import type { + SomeCompartmentDescriptor, CompartmentDescriptor, CompartmentMapDescriptor, DigestedCompartmentMapDescriptor, @@ -735,37 +736,51 @@ interface BaseParserImplementation { heuristicImports: boolean; } -export interface ParserImplementation extends BaseParserImplementation { - parse: ParseFn; +export interface ParserImplementation< + TCompartmentDescriptor extends + SomeCompartmentDescriptor = SomeCompartmentDescriptor, +> extends BaseParserImplementation { + parse: ParseFn; synchronous: true; } -export interface AsyncParserImplementation extends BaseParserImplementation { - parse: AsyncParseFn; +export interface AsyncParserImplementation< + TCompartmentDescriptor extends + SomeCompartmentDescriptor = SomeCompartmentDescriptor, +> extends BaseParserImplementation { + parse: AsyncParseFn; synchronous: false; } /** * Options bag for a {@link ParseFn} or {@link AsyncParseFn}. + * + * @template TCompartmentDescriptor The compartment descriptor to use for the parse */ -export type ParseOptions = Partial<{ +export type ParseOptions< + TCompartmentDescriptor extends + SomeCompartmentDescriptor = SomeCompartmentDescriptor, +> = Partial<{ sourceMap: string | undefined; sourceMapHook: ParseSourceMapHook | undefined; sourceMapUrl: string | undefined; readPowers: ReadFn | ReadPowers | undefined; - compartmentDescriptor: CompartmentDescriptor | undefined; + compartmentDescriptor: TCompartmentDescriptor | undefined; }> & ArchiveOnlyOption; /** * Arguments for a {@link ParseFn} or {@link AsyncParseFn}. */ -export type ParseArguments = [ +export type ParseArguments< + TCompartmentDescriptor extends + SomeCompartmentDescriptor = SomeCompartmentDescriptor, +> = [ bytes: Uint8Array, specifier: string, moduleLocation: string, packageLocation: string, - options?: ParseOptions, + options?: ParseOptions, ]; /** @@ -788,17 +803,17 @@ export type ParseResult = { * Because {@link ParseResult} contains {@link FinalStaticModuleType} from * `ses`, those types would want to be moved out of `ses` with it. */ -export type ParseFn = { isSyncParser?: true } & (( - ...args: ParseArguments -) => ParseResult); +export interface ParseFn< + TCompartmentDescriptor extends + SomeCompartmentDescriptor = SomeCompartmentDescriptor, +> { + isSyncParser?: true; + (...args: ParseArguments): ParseResult; +} /** * An asynchronous module parsing function. */ -export type AsyncParseFn = { isSyncParser?: false } & (( - ...args: ParseArguments -) => Promise); - /** * Mapping of `Language` to synchronous {@link ParserImplementation}s only. * @@ -808,6 +823,13 @@ export type SyncParserForLanguage = Record< Language | string, ParserImplementation >; +export interface AsyncParseFn< + TCompartmentDescriptor extends + SomeCompartmentDescriptor = SomeCompartmentDescriptor, +> { + isSyncParser?: false; + (...args: ParseArguments): Promise; +} /** * Mapping of `Language` to {@link ParserImplementation diff --git a/packages/compartment-mapper/src/types/policy-schema.ts b/packages/compartment-mapper/src/types/policy-schema.ts index 6189cfe467..636f2d3bac 100644 --- a/packages/compartment-mapper/src/types/policy-schema.ts +++ b/packages/compartment-mapper/src/types/policy-schema.ts @@ -6,6 +6,7 @@ */ import type { WILDCARD_POLICY_VALUE } from '../policy-format.js'; +import type { IsAny } from './typescript.js'; /* eslint-disable no-use-before-define */ @@ -31,9 +32,16 @@ export type ImplicitAttenuationDefinition = [any, ...any[]]; export type AttenuationDefinition = | FullAttenuationDefinition | ImplicitAttenuationDefinition; + +/** + * Information about the attenuator implementation + */ export type UnifiedAttenuationDefinition = { + /** Name of the attenuator (for error messages) */ displayName: string; + /** The module specifier of the implementation */ specifier: string | null; + /** Parameters to pass to the attenuator at invocation */ params?: any[] | undefined; }; @@ -52,10 +60,24 @@ export type PropertyPolicy = Record; * A type representing a policy item, which can be a {@link WildcardPolicy * wildcard policy}, a property policy, `undefined`, or defined by an * attenuator + * + * @remarks + * The void-vs-custom `T` branch was originally `[T] extends [void] ? … : …`, but + * the type `any` also makes that test succeed, so `PolicyItem` used to + * reduce to the same as `void` and + * `PackagePolicy = AnyPackagePolicy` was not a supertype of + * policies with extra string literals (for example, LavaMoat's {@code "root"} on + * package imports). A separate branch for a wide + * `any` type parameter yields + * `PolicyItem = WildcardPolicy | PropertyPolicy | any` so + * `AnyPackagePolicy` correctly accepts all package policy item shapes. */ -export type PolicyItem = [T] extends [void] - ? WildcardPolicy | PropertyPolicy - : WildcardPolicy | PropertyPolicy | T; +export type PolicyItem = + IsAny extends true + ? WildcardPolicy | PropertyPolicy | T + : [T] extends [void] + ? WildcardPolicy | PropertyPolicy + : WildcardPolicy | PropertyPolicy | T; /** * An object representing a nested attenuation definition. @@ -69,10 +91,10 @@ export type NestedAttenuationDefinition = Record< * An object representing a base package policy. */ export type PackagePolicy< - PackagePolicyItem = void, - GlobalsPolicyItem = void, - BuiltinsPolicyItem = void, - ExtraOptions = unknown, + PackagePolicyExtra = void, + GlobalsPolicyExtra = void, + BuiltinsPolicyExtra = void, + Options = unknown, > = { /** * The default attenuator, if any. @@ -81,17 +103,17 @@ export type PackagePolicy< /** * The policy item for packages. */ - packages?: PolicyItem | undefined; + packages?: PolicyItem | undefined; /** * The policy item or full attenuation definition for globals. */ - globals?: AttenuationDefinition | PolicyItem | undefined; + globals?: AttenuationDefinition | PolicyItem | undefined; /** * The policy item or nested attenuation definition for builtins. */ builtins?: | NestedAttenuationDefinition - | PolicyItem + | PolicyItem | undefined; /** * Whether to disable global freeze. @@ -104,26 +126,26 @@ export type PackagePolicy< /** * Any additional user-defined options can be added to the policy here */ - options?: ExtraOptions | undefined; + options?: Options | undefined; }; /** * An object representing a base policy. */ export type Policy< - PackagePolicyItem = void, - GlobalsPolicyItem = void, - BuiltinsPolicyItem = void, - ExtraOptions = unknown, + PackagePolicyExtra = void, + GlobalsPolicyExtra = void, + BuiltinsPolicyExtra = void, + Options = unknown, > = { /** The package policies for the resources. */ resources: Record< string, PackagePolicy< - PackagePolicyItem, - GlobalsPolicyItem, - BuiltinsPolicyItem, - ExtraOptions + PackagePolicyExtra, + GlobalsPolicyExtra, + BuiltinsPolicyExtra, + Options > >; /** The default attenuator. */ @@ -131,16 +153,20 @@ export type Policy< /** The package policy for the entry. */ entry?: | PackagePolicy< - PackagePolicyItem, - GlobalsPolicyItem, - BuiltinsPolicyItem, - ExtraOptions + PackagePolicyExtra, + GlobalsPolicyExtra, + BuiltinsPolicyExtra, + Options > | undefined; }; -/** Any {@link Policy} */ +/** + * Any {@link Policy} + */ export type SomePolicy = Policy; -/** Any {@link PackagePolicy} */ -export type SomePackagePolicy = PackagePolicy; +/** + * Any {@link PackagePolicy} + */ +export type SomePackagePolicy = PackagePolicy; diff --git a/packages/compartment-mapper/src/types/typescript.ts b/packages/compartment-mapper/src/types/typescript.ts index 871a6b4c31..3379e72e9d 100644 --- a/packages/compartment-mapper/src/types/typescript.ts +++ b/packages/compartment-mapper/src/types/typescript.ts @@ -77,3 +77,11 @@ export type UnionToIntersection = ( * Makes a nicer tooltip for `T` in IDEs (most of the time). */ export type Simplify = { [K in keyof T]: T[K] } & {}; + +/** + * `true` when the type parameter is a wide `any` + * + * `0 extends 1 & T` is true for `T = any` and false for `void` and specific literals + * @see {@link https://github.com/microsoft/TypeScript/issues/30029} + */ +export type IsAny = 0 extends 1 & T ? true : false; diff --git a/packages/module-source/src/external.types.d.ts b/packages/module-source/src/external.types.d.ts index fd9f79c3da..a578bf2db6 100644 --- a/packages/module-source/src/external.types.d.ts +++ b/packages/module-source/src/external.types.d.ts @@ -1,3 +1,4 @@ +export type * from './types/visitor-passes.ts'; export type { SourceMapObject, SourceMapHook, From c130a92a8be1a13d33f242b7c3792195917828e5 Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Wed, 15 Apr 2026 17:58:24 -0700 Subject: [PATCH 2/2] feat(module-source): parser pipeline support This exposes a new function, `createModuleSourcePasses()` for use with `@endo/parser-pipeline`. Refactored code for reuse between `createModuleSourcePasses()` and `ModuleSource()` --- .changeset/ripe-showers-remain.md | 5 + packages/module-source/index.js | 2 + packages/module-source/src/babel-plugin.js | 12 +- packages/module-source/src/functor.js | 126 +++++++++++++++ packages/module-source/src/module-source.js | 51 ++---- .../module-source/src/transform-analyze.js | 104 +++--------- .../module-source/src/types/visitor-passes.ts | 119 ++++++++++++++ packages/module-source/src/visitor-passes.js | 148 ++++++++++++++++++ .../test/_benchmark-babel-plugin.js | 14 +- .../module-source/test/visitor-passes.test.js | 100 ++++++++++++ 10 files changed, 543 insertions(+), 138 deletions(-) create mode 100644 .changeset/ripe-showers-remain.md create mode 100644 packages/module-source/src/functor.js create mode 100644 packages/module-source/src/types/visitor-passes.ts create mode 100644 packages/module-source/src/visitor-passes.js create mode 100644 packages/module-source/test/visitor-passes.test.js diff --git a/.changeset/ripe-showers-remain.md b/.changeset/ripe-showers-remain.md new file mode 100644 index 0000000000..6cdb50335a --- /dev/null +++ b/.changeset/ripe-showers-remain.md @@ -0,0 +1,5 @@ +--- +'@endo/module-source': minor +--- + +Expose `createModuleSourcePasses()` for use with `@endo/parser-pipeline`. diff --git a/packages/module-source/index.js b/packages/module-source/index.js index a9e4dac9a1..f9d8d43848 100644 --- a/packages/module-source/index.js +++ b/packages/module-source/index.js @@ -2,3 +2,5 @@ export * from './src/external.types.js'; export { ModuleSource } from './src/module-source.js'; + +export { createModuleSourcePasses } from './src/visitor-passes.js'; diff --git a/packages/module-source/src/babel-plugin.js b/packages/module-source/src/babel-plugin.js index 1ed7bff4b8..0669e08d86 100644 --- a/packages/module-source/src/babel-plugin.js +++ b/packages/module-source/src/babel-plugin.js @@ -2,6 +2,10 @@ import * as h from './hidden.js'; +/** + * @import {PluginFactory, TransformSourceParams} from './types/module-source.js' + */ + /* * Collects all of the identifiers on the left-hand-side of an exported * assignment expression, deeply exploring complex destructuring assignment. @@ -40,6 +44,12 @@ const collectPatternIdentifiers = (path, pattern) => { } }; +/** + * Creates a plugin for the Babel parser that analyzes and transforms module source code. + * + * @param {TransformSourceParams} options + * @returns {{analyzePlugin: PluginFactory, transformPlugin: PluginFactory}} + */ function makeModulePlugins(options) { const { sourceType, @@ -209,7 +219,7 @@ function makeModulePlugins(options) { } else { // Rewrite to be just name = value. soften(id); - options.hoistedDecls.push([name]); + options.hoistedDecls.push([name, false, undefined]); replacements.push( t.expressionStatement( t.assignmentExpression( diff --git a/packages/module-source/src/functor.js b/packages/module-source/src/functor.js new file mode 100644 index 0000000000..45503b6f0a --- /dev/null +++ b/packages/module-source/src/functor.js @@ -0,0 +1,126 @@ +/** + * Shared logic for building the SES functor source string and assembling + * the frozen module record (see {@link ModuleSourceRecord}). + * + * @module + */ + +import * as h from './hidden.js'; + +/** + * @import {ModuleSourceRecord} from './types/module-source.js' + */ + +const { keys, values, freeze: objectFreeze } = Object; + +/** + * It's {@link objectFreeze | Object.freeze} without the `Readonly`! + * + * @privateRemarks Remove this if we decide to make the fields of `PrecompiledModuleSource` readonly. + */ +const freeze = /** @type {(v: T) => T} */ (objectFreeze); + +const { stringify: js } = JSON; + +/** + * Builds the SES functor source string from generated code and the analysis + * state accumulated during the analyzer and transform passes. + * + * Constructs the `$h_imports([...])` preamble from `sourceOptions.importSources`, + * `importDecls`, and `hoistedDecls`, then wraps `scriptSource` in the SES + * functor calling convention. + * + * @param {string} scriptSource The code produced by `@babel/generator` + * after all transform passes. + * @param {Record} sourceOptions The mutable state bag populated + * by `makeModulePlugins`. + * @param {string} [sourceUrl] The source URL for the module. + * @returns {string} The functor source string. + */ +export const buildFunctorSource = (scriptSource, sourceOptions, sourceUrl) => { + const isrc = sourceOptions.importSources; + + let preamble = sourceOptions.importDecls.join(','); + if (preamble !== '') { + preamble = `let ${preamble};`; + } + + preamble += `${h.HIDDEN_IMPORTS}([${keys(isrc) + .map( + src => + `[${js(src)}, [${Object.entries(isrc[src]) + .map(([exp, upds]) => `[${js(exp)},[${upds.join(',')}]]`) + .join(',')}]]`, + ) + .join(',')}]);`; + + preamble += sourceOptions.hoistedDecls + .map(([vname, isOnce, cvname]) => { + let src = ''; + if (cvname) { + src = `Object.defineProperty(${cvname},'name',{value:${js(vname)}});`; + } + const hDeclId = isOnce ? h.HIDDEN_ONCE : h.HIDDEN_LIVE; + src += `${hDeclId}.${vname}(${cvname || ''});`; + return src; + }) + .join(''); + + let functorSource = `\ +({imports:${h.HIDDEN_IMPORTS},liveVar:${h.HIDDEN_LIVE},onceVar:${h.HIDDEN_ONCE},import:${h.HIDDEN_IMPORT},importMeta:${h.HIDDEN_META}})=>(function(){'use strict';\ +${preamble}\ +${scriptSource} +})() +`; + + if (sourceUrl) { + functorSource += `//# sourceURL=${sourceUrl}\n`; + } + + return functorSource; +}; + +/** + * Computes the derived arrays (`imports`, `exports`, `reexports`) from the + * raw maps in `sourceOptions` and assembles a {@link ModuleSourceRecord}. + * + * @param {Record} sourceOptions The mutable state bag populated + * by `makeModulePlugins`. + * @param {string} functorSource The functor source string from + * {@link buildFunctorSource}. + * @returns {ModuleSourceRecord} + */ +export const buildModuleRecord = (sourceOptions, functorSource) => { + for (const entry of values(sourceOptions.liveExportMap)) { + freeze(entry); + } + for (const entry of values(sourceOptions.fixedExportMap)) { + freeze(entry); + } + for (const reexports of values(sourceOptions.reexportMap)) { + for (const pair of reexports) { + freeze(pair); + } + freeze(reexports); + } + + return freeze({ + imports: freeze([...keys(sourceOptions.imports)]), + exports: freeze( + [ + ...keys(sourceOptions.liveExportMap), + ...keys(sourceOptions.fixedExportMap), + ...values(sourceOptions.reexportMap) + .flat() + .map(([_, exportName]) => exportName), + ].sort(), + ), + reexports: freeze([...sourceOptions.exportAlls].sort()), + __syncModuleProgram__: functorSource, + __liveExportMap__: freeze(sourceOptions.liveExportMap), + __fixedExportMap__: freeze(sourceOptions.fixedExportMap), + __reexportMap__: freeze(sourceOptions.reexportMap), + __needsImport__: sourceOptions.dynamicImport.present, + __needsImportMeta__: sourceOptions.importMeta.present, + }); +}; diff --git a/packages/module-source/src/module-source.js b/packages/module-source/src/module-source.js index de8d62b33a..917d1bfe2d 100644 --- a/packages/module-source/src/module-source.js +++ b/packages/module-source/src/module-source.js @@ -57,45 +57,18 @@ export function ModuleSource(source, opts = {}) { if (typeof opts === 'string') { opts = { sourceUrl: opts }; } - const { - imports, - functorSource, - liveExportMap, - reexportMap, - fixedExportMap, - exportAlls, - needsImport, - needsImportMeta, - } = analyzeModule(source, opts); - this.imports = freeze([...keys(imports)]); - this.exports = freeze( - [ - ...keys(liveExportMap), - ...keys(fixedExportMap), - ...values(reexportMap) - .flat() - .map(([_, exportName]) => exportName), - ].sort(), - ); - this.reexports = freeze([...exportAlls].sort()); - this.__syncModuleProgram__ = functorSource; - for (const entry of values(liveExportMap)) { - freeze(entry); - } - for (const entry of values(fixedExportMap)) { - freeze(entry); - } - for (const reexports of values(reexportMap)) { - for (const pair of reexports) { - freeze(pair); - } - freeze(reexports); - } - this.__liveExportMap__ = freeze(liveExportMap); - this.__reexportMap__ = freeze(reexportMap); - this.__fixedExportMap__ = freeze(fixedExportMap); - this.__needsImport__ = needsImport; - this.__needsImportMeta__ = needsImportMeta; + // analyzeModule now returns a frozen PrecompiledModuleSource-shaped record + // via buildModuleRecord(), so we copy its properties directly. + const record = analyzeModule(source, opts); + this.imports = record.imports; + this.exports = record.exports; + this.reexports = record.reexports; + this.__syncModuleProgram__ = record.__syncModuleProgram__; + this.__liveExportMap__ = record.__liveExportMap__; + this.__reexportMap__ = record.__reexportMap__; + this.__fixedExportMap__ = record.__fixedExportMap__; + this.__needsImport__ = record.__needsImport__; + this.__needsImportMeta__ = record.__needsImportMeta__; freeze(this); } diff --git a/packages/module-source/src/transform-analyze.js b/packages/module-source/src/transform-analyze.js index 9f918ee6a9..1705c57c4e 100644 --- a/packages/module-source/src/transform-analyze.js +++ b/packages/module-source/src/transform-analyze.js @@ -3,48 +3,28 @@ import { makeTransformSource } from './transform-source.js'; import makeModulePlugins from './babel-plugin.js'; import * as h from './hidden.js'; +import { createSourceOptions } from './source-options.js'; +import { buildFunctorSource, buildModuleRecord } from './functor.js'; -const { freeze } = Object; - -/** @import {Options} from './module-source.js' */ +/** @import {ModuleSourceOptions} from './types/module-source.js' */ const makeCreateStaticRecord = transformSource => /** * * @param {string} moduleSource - * @param {Options} options + * @param {ModuleSourceOptions} options */ function createStaticRecord( moduleSource, { sourceUrl, sourceMapUrl, sourceMap, sourceMapHook } = {}, ) { - // Transform the Module source code. - const sourceOptions = { + const sourceOptions = createSourceOptions({ sourceUrl, sourceMap, sourceMapUrl, sourceMapHook, - sourceType: 'module', - // exportNames of variables that are only initialized and used, but - // never assigned to. - fixedExportMap: Object.create(null), - // Record of imported module specifier names to list of importNames. - // The importName '*' is that module's module namespace object. - imports: Object.create(null), - // List of module specifiers that we export all from. - exportAlls: [], - // exportNames of variables that are assigned to, or reexported and - // therefore assumed live. A reexported variable might not have any - // localName. - reexportMap: Object.create(null), - liveExportMap: Object.create(null), - hoistedDecls: [], - importSources: Object.create(null), - importDecls: [], - dynamicImport: { present: false }, - // enables passing import.meta usage hints up. - importMeta: { present: false }, - }; + }); + if (moduleSource.startsWith('#!')) { // Comment out the shebang lines. moduleSource = `//${moduleSource}`; @@ -62,71 +42,23 @@ const makeCreateStaticRecord = transformSource => ); } - let preamble = sourceOptions.importDecls.join(','); - if (preamble !== '') { - preamble = `let ${preamble};`; - } - const js = JSON.stringify; - const isrc = sourceOptions.importSources; - preamble += `${h.HIDDEN_IMPORTS}([${Object.keys(isrc) - .map( - src => - `[${js(src)}, [${Object.entries(isrc[src]) - .map(([exp, upds]) => `[${js(exp)},[${upds.join(',')}]]`) - .join(',')}]]`, - ) - .join(',')}]);`; - preamble += sourceOptions.hoistedDecls - .map(([vname, isOnce, cvname]) => { - let src = ''; - if (cvname) { - // It's a function assigned to, so set its name property. - src = `Object.defineProperty(${cvname},'name',{value:${js(vname)}});`; - } - const hDeclId = isOnce ? h.HIDDEN_ONCE : h.HIDDEN_LIVE; - src += `${hDeclId}.${vname}(${cvname || ''});`; - return src; - }) - .join(''); - - // The outer function destructures the module calling convention's internal - // variables into hidden lexical variables. - // The inner function binds `this` to `undefined` and overshadows the - // evaluator's `arguments` with a completely empty `arguments` object. - // There is no avoiding the overshadowing of `globalThis.arguments` if it - // exists in this emulation of ESM since the evaluator binds `arguments` as - // well. - // Relies on the evaluator to ensure these functions are strict. - let functorSource = `\ -({imports:${h.HIDDEN_IMPORTS},liveVar:${h.HIDDEN_LIVE},onceVar:${h.HIDDEN_ONCE},import:${h.HIDDEN_IMPORT},importMeta:${h.HIDDEN_META}})=>(function(){'use strict';\ -${preamble}\ -${scriptSource} -})() -`; + const functorSource = buildFunctorSource( + scriptSource, + sourceOptions, + sourceUrl, + ); - if (sourceUrl) { - functorSource += `//# sourceURL=${sourceUrl}\n`; - } - const moduleAnalysis = freeze({ - exportAlls: freeze(sourceOptions.exportAlls), - imports: freeze(sourceOptions.imports), - liveExportMap: freeze(sourceOptions.liveExportMap), - fixedExportMap: freeze(sourceOptions.fixedExportMap), - reexportMap: freeze(sourceOptions.reexportMap), - needsImport: sourceOptions.dynamicImport.present, - needsImportMeta: sourceOptions.importMeta.present, - functorSource, - }); - return moduleAnalysis; + return buildModuleRecord(sourceOptions, functorSource); }; -export const makeModuleAnalyzer = babel => { - const transformSource = makeTransformSource(makeModulePlugins, babel); +export const makeModuleAnalyzer = () => { + const transformSource = makeTransformSource(makeModulePlugins); return makeCreateStaticRecord(transformSource); }; -export const makeModuleTransformer = (babel, importer) => { - const transformSource = makeTransformSource(makeModulePlugins, babel); +// TODO: May be unused; referenced only in ses/test262 +export const makeModuleTransformer = (_babel, importer) => { + const transformSource = makeTransformSource(makeModulePlugins); const createStaticRecord = makeCreateStaticRecord(transformSource); return { rewrite(ss) { diff --git a/packages/module-source/src/types/visitor-passes.ts b/packages/module-source/src/types/visitor-passes.ts new file mode 100644 index 0000000000..b1ed223537 --- /dev/null +++ b/packages/module-source/src/types/visitor-passes.ts @@ -0,0 +1,119 @@ +/** + * Types for {@link createModuleSourcePasses} and + * {@link createCjsModuleSourcePasses}. + * + * @module + */ + +import type { Visitor } from '@babel/traverse'; +import type { FinalStaticModuleType } from 'ses'; +import type { CjsModuleSourceRecord } from './module-source.js'; + +/** + * A read-only analysis pass compatible with `@endo/parser-pipeline`. + * + * @template TResult - The type of data this analyzer produces. + */ +export interface AnalyzerPass { + readonly visitor: Visitor; + getResults(): TResult; +} + +/** + * A mutating transform pass compatible with `@endo/parser-pipeline`. + */ +export interface TransformPass { + readonly visitor: Visitor; +} + +/** + * Results of module-source static analysis. + */ +export interface ModuleSourceAnalysis { + /** Module specifiers imported by this module. */ + imports: string[]; + + /** All exported names (live, fixed, and re-exported), sorted. */ + exports: string[]; + + /** Module specifiers that are re-exported via `export * from`. */ + reexports: string[]; + + liveExportMap: Record; + fixedExportMap: Record; + reexportMap: Record; + + /** Whether `import()` is used dynamically. */ + needsImport: boolean; + + /** Whether `import.meta` is referenced. */ + needsImportMeta: boolean; +} + +/** + * Return type of {@link createModuleSourcePasses}. + */ +export interface ModuleSourcePassesResult { + /** Read-only analysis visitor that extracts imports, exports, and reexports. */ + analyzerPass: AnalyzerPass; + + /** Mutating visitor that rewrites ESM to a SES-compatible script functor. */ + transformPass: TransformPass; + + /** + * Constructs a `ModuleSource`-compatible record from generated code and the + * analysis state accumulated during the analyzer and transform passes. + * + * @param transformedSource - The code produced by `@babel/generator`. + * @param sourceUrl - The source URL for the module. + */ + buildRecord( + transformedSource: string, + sourceUrl?: string, + ): FinalStaticModuleType; +} + +// ---- CJS types ---- + +/** + * Results of CJS module-source static analysis. + */ +export interface CjsModuleAnalysis { + /** Specifiers from `require()` calls. */ + requires: string[]; + + /** Export names (filtered for valid JS identifiers, minus unsafe getters). */ + exports: string[]; + + /** Specifiers that are wholesale reexported. */ + reexports: string[]; + + /** Specifiers from `import()` calls. */ + imports: string[]; + + /** Whether any `import()` calls were found. */ + needsImport: boolean; +} + +/** + * Return type of {@link createCjsModuleSourcePasses}. + */ +export interface CjsModuleSourcePassesResult { + /** Read-only analysis visitor that extracts CJS imports, exports, and reexports. */ + analyzerPass: AnalyzerPass; + + /** Mutating visitor that rewrites `import()` to the SES hidden identifier. */ + transformPass: TransformPass; + + /** + * Constructs a `CjsModuleSourceRecord` from generated code and the analysis + * state accumulated during the analyzer and transform passes. + * + * @param transformedSource - The code produced by `@babel/generator`. + * @param sourceUrl - The source URL for the module. + */ + buildRecord( + transformedSource: string, + sourceUrl?: string, + ): CjsModuleSourceRecord; +} diff --git a/packages/module-source/src/visitor-passes.js b/packages/module-source/src/visitor-passes.js new file mode 100644 index 0000000000..1b304f6948 --- /dev/null +++ b/packages/module-source/src/visitor-passes.js @@ -0,0 +1,148 @@ +/** + * Provide functions which create paired analyzer and transform visitor passes + * for use with `@endo/parser-pipeline`. + * + * @module + */ + +/** + * @import {ModuleSourcePassesResult} from './types/visitor-passes.js' + * @import {CjsModuleSourcePassesResult} from './types/visitor-passes.js' + * @import {ModuleSourceRecord} from './types/module-source.js' + * @import {CjsModuleSourceRecord} from './types/module-source.js' + */ + +import makeModulePlugins from './babel-plugin.js'; +import makeCjsModulePlugins from './cjs-babel-plugin.js'; +import { + createSourceOptions, + createCjsSourceOptions, +} from './source-options.js'; +import { buildFunctorSource, buildModuleRecord } from './functor.js'; +import { buildCjsFunctorSource, buildCjsModuleRecord } from './cjs-functor.js'; +import { visitorFromPlugin } from './plugin-util.js'; + +/** + * Creates paired analyzer and transform visitor passes for module-source + * analysis, plus a `buildRecord` function that constructs the final module + * record from generated code. + * + * Must be called once per module to get fresh state. The returned analyzer and + * transform passes share internal state and must be used in order: analyzer + * first, then transform. + * + * @param {object} [options] + * @param {boolean} [options.allowHidden] - Allow hidden identifier usage. + * @returns {ModuleSourcePassesResult} + */ +export const createModuleSourcePasses = (options = {}) => { + const { allowHidden = false } = options; + + const sourceOptions = createSourceOptions({ allowHidden }); + + const { analyzePlugin, transformPlugin } = makeModulePlugins(sourceOptions); + + const analyzerPass = { + visitor: visitorFromPlugin(analyzePlugin), + getResults() { + const { keys, values } = Object; + return { + imports: keys(sourceOptions.imports), + exports: [ + ...keys(sourceOptions.liveExportMap), + ...keys(sourceOptions.fixedExportMap), + ...values(sourceOptions.reexportMap) + .flat() + .map(([_, exportName]) => exportName), + ].sort(), + reexports: [...sourceOptions.exportAlls].sort(), + liveExportMap: sourceOptions.liveExportMap, + fixedExportMap: sourceOptions.fixedExportMap, + reexportMap: sourceOptions.reexportMap, + needsImport: sourceOptions.dynamicImport.present, + needsImportMeta: sourceOptions.importMeta.present, + }; + }, + }; + + const transformPass = { + visitor: visitorFromPlugin(transformPlugin), + }; + + /** + * Constructs a `ModuleSource`-compatible record from the generated code and + * the analysis state accumulated during the analyzer and transform passes. + * + * @param {string} source - The code produced by `@babel/generator` + * after all transform passes. + * @param {string} location - The source URL for the module. + * @returns {ModuleSourceRecord} A `ModuleSource`-compatible object + */ + const buildRecord = (source, location) => { + const functorSource = buildFunctorSource(source, sourceOptions, location); + return buildModuleRecord(sourceOptions, functorSource); + }; + + return { analyzerPass, transformPass, buildRecord }; +}; + +/** + * Creates paired analyzer and transform visitor passes for CJS module-source + * analysis, plus a `buildRecord` function that constructs the final CJS module + * record from generated code. + * + * Must be called once per module to get fresh state. The returned analyzer and + * transform passes share internal state and must be used in order: analyzer + * first, then transform. + * + * @param {object} [options] + * @param {boolean} [options.allowHidden] - Allow hidden identifier usage. + * @returns {CjsModuleSourcePassesResult} + */ +export const createCjsModuleSourcePasses = (options = {}) => { + const { allowHidden = false } = options; + + const sourceOptions = createCjsSourceOptions({ allowHidden }); + + const { analyzePlugin, transformPlugin } = + makeCjsModulePlugins(sourceOptions); + + const analyzerPass = { + visitor: visitorFromPlugin(analyzePlugin), + getResults() { + return { + requires: [...sourceOptions.requires], + exports: [...sourceOptions.exports].filter( + name => !sourceOptions.unsafeGetters.has(name), + ), + reexports: [...sourceOptions.reexports], + imports: [...sourceOptions.imports], + needsImport: sourceOptions.dynamicImport.present, + }; + }, + }; + + const transformPass = { + visitor: visitorFromPlugin(transformPlugin), + }; + + /** + * Constructs a `CjsModuleSourceRecord` from the generated code and the + * analysis state accumulated during the analyzer and transform passes. + * + * @param {string} source - The code produced by `@babel/generator` + * after all transform passes. + * @param {string} location - The source URL for the module. + * @returns {CjsModuleSourceRecord} + */ + const buildRecord = (source, location) => { + const functorSource = buildCjsFunctorSource( + source, + sourceOptions, + location, + ); + return buildCjsModuleRecord(sourceOptions, functorSource); + }; + + return { analyzerPass, transformPass, buildRecord }; +}; diff --git a/packages/module-source/test/_benchmark-babel-plugin.js b/packages/module-source/test/_benchmark-babel-plugin.js index 1874179933..2f8e679d52 100644 --- a/packages/module-source/test/_benchmark-babel-plugin.js +++ b/packages/module-source/test/_benchmark-babel-plugin.js @@ -3,6 +3,7 @@ import fs from 'fs'; import url from 'url'; import { makeTransformSource } from '../src/transform-source.js'; import makeModulePlugins from '../src/babel-plugin.js'; +import { createSourceOptions } from '../src/source-options.js'; const suite = new Benchmark.Suite(); @@ -23,21 +24,10 @@ const cases = [ ]; const transformSource = makeTransformSource(makeModulePlugins); -const freshOptions = () => ({ - sourceType: 'module', - fixedExportMap: Object.create(null), - imports: Object.create(null), - exportAlls: [], - liveExportMap: Object.create(null), - hoistedDecls: [], - importSources: Object.create(null), - importDecls: [], - importMeta: { present: false }, -}); cases.map(testCase => suite.add(testCase.name, () => { - transformSource(testCase.fixture, freshOptions()); + transformSource(testCase.fixture, createSourceOptions()); }), ); diff --git a/packages/module-source/test/visitor-passes.test.js b/packages/module-source/test/visitor-passes.test.js new file mode 100644 index 0000000000..7abc01217b --- /dev/null +++ b/packages/module-source/test/visitor-passes.test.js @@ -0,0 +1,100 @@ +/* eslint-disable no-underscore-dangle */ +import test from '@endo/ses-ava/prepare-endo.js'; +import { parse as parseBabel } from '@babel/parser'; +import babelTraverse from '@babel/traverse'; +import { generate as generateBabel } from '@babel/generator'; +import { createModuleSourcePasses } from '../src/visitor-passes.js'; +import { ModuleSource } from '../src/module-source.js'; + +const { default: traverseBabel } = babelTraverse; + +test('createModuleSourcePasses returns analyzerPass, transformPass, and buildRecord', t => { + const passes = createModuleSourcePasses(); + t.is(typeof passes.analyzerPass, 'object'); + t.is(typeof passes.analyzerPass.visitor, 'object'); + t.is(typeof passes.analyzerPass.getResults, 'function'); + t.is(typeof passes.transformPass, 'object'); + t.is(typeof passes.transformPass.visitor, 'object'); + t.is(typeof passes.buildRecord, 'function'); +}); + +test('visitor passes produce same imports/exports as ModuleSource', t => { + const source = ` + import { foo } from 'bar'; + import * as baz from 'qux'; + export const x = foo(); + export { baz }; + `; + + const ms = new ModuleSource(source); + + const passes = createModuleSourcePasses(); + const ast = parseBabel(source, { + sourceType: 'module', + tokens: true, + createParenthesizedExpressions: true, + }); + + traverseBabel(ast, passes.analyzerPass.visitor); + traverseBabel(ast, passes.transformPass.visitor); + + const analysis = passes.analyzerPass.getResults(); + + t.deepEqual([...analysis.imports].sort(), [...ms.imports].sort()); + t.deepEqual([...analysis.exports].sort(), [...ms.exports].sort()); + t.deepEqual([...analysis.reexports].sort(), [...ms.reexports].sort()); +}); + +test('buildRecord produces a record with __syncModuleProgram__', t => { + const source = `import { foo } from 'bar'; export const x = foo();`; + + const passes = createModuleSourcePasses(); + const ast = parseBabel(source, { + sourceType: 'module', + tokens: true, + createParenthesizedExpressions: true, + }); + + traverseBabel(ast, passes.analyzerPass.visitor); + traverseBabel(ast, passes.transformPass.visitor); + + const { code } = generateBabel( + ast, + { + // @ts-expect-error undocumented + experimental_preserveFormat: true, + preserveFormat: true, + retainLines: true, + verbatim: true, + }, + source, + ); + + const record = /** @type {any} */ ( + passes.buildRecord(code, 'file:///test.js') + ); + + t.truthy(record.__syncModuleProgram__); + t.true(typeof record.__syncModuleProgram__ === 'string'); + t.truthy(record.imports); + t.truthy(record.exports); + t.truthy(record.__fixedExportMap__); +}); + +test('re-exports are correctly detected', t => { + const source = `export * from 'reexported-module';`; + + const ms = new ModuleSource(source); + + const passes = createModuleSourcePasses(); + const ast = parseBabel(source, { + sourceType: 'module', + tokens: true, + createParenthesizedExpressions: true, + }); + + traverseBabel(ast, passes.analyzerPass.visitor); + const analysis = passes.analyzerPass.getResults(); + + t.deepEqual(analysis.reexports, ms.reexports); +});