diff --git a/Makefile b/Makefile index c22e5ea..403c3e9 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c @mkdir -p lib ../wasi-sdk-11.0/bin/clang src/lexer.c -I include-wasm --sysroot=../wasi-sdk-11.0/share/wasi-sysroot -o lib/lexer.wasm -nostartfiles \ -Wl,-z,stack-size=13312,--no-entry,--compress-relocations,--strip-all,--export=__heap_base,\ - --export=parseCJS,--export=sa,--export=e,--export=re,--export=es,--export=ee,--export=rre,--export=ree,--export=res,--export=ree \ + --export=parseCJS,--export=sa,--export=e,--export=re,--export=es,--export=ee,--export=rre,--export=ree,--export=res,--export=ree,--export=rrq,--export=rqs,--export=rqe \ -Wno-logical-op-parentheses -Wno-parentheses \ -Oz diff --git a/include-wasm/cjs-module-lexer.h b/include-wasm/cjs-module-lexer.h index 10c823b..a3af310 100755 --- a/include-wasm/cjs-module-lexer.h +++ b/include-wasm/cjs-module-lexer.h @@ -30,6 +30,9 @@ Slice* export_write_head = NULL; Slice* first_reexport = NULL; Slice* reexport_read_head = NULL; Slice* reexport_write_head = NULL; +Slice* first_require = NULL; +Slice* require_read_head = NULL; +Slice* require_write_head = NULL; void* analysis_base; void* analysis_head; @@ -48,6 +51,9 @@ const uint16_t* sa (uint32_t utf16Len) { first_reexport = NULL; reexport_write_head = NULL; reexport_read_head = NULL; + first_require = NULL; + require_write_head = NULL; + require_read_head = NULL; return source; } @@ -72,6 +78,14 @@ uint32_t res () { uint32_t ree () { return reexport_read_head->end - source; } +// getRequireStart +uint32_t rqs () { + return require_read_head->start - source; +} +// getRequireEnd +uint32_t rqe () { + return require_read_head->end - source; +} // readExport bool re () { if (export_read_head == NULL) @@ -92,6 +106,16 @@ bool rre () { return false; return true; } +// readRequire +bool rrq () { + if (require_read_head == NULL) + require_read_head = first_require; + else + require_read_head = require_read_head->next; + if (require_read_head == NULL) + return false; + return true; +} bool parse (uint32_t point); @@ -119,9 +143,28 @@ void _addReexport (const uint16_t* start, const uint16_t* end) { reexport->end = end; reexport->next = NULL; } +void _addRequire (const uint16_t* start, const uint16_t* end) { + Slice* require = (Slice*)(analysis_head); + analysis_head = analysis_head + sizeof(Slice); + if (require_write_head == NULL) + first_require = require; + else + require_write_head->next = require; + require_write_head = require; + require->start = start; + require->end = end; + require->next = NULL; +} void (*addExport)(const uint16_t*, const uint16_t*) = &_addExport; void (*addReexport)(const uint16_t*, const uint16_t*) = &_addReexport; -bool parseCJS (uint16_t* source, uint32_t sourceLen, void (*addExport)(const uint16_t* start, const uint16_t* end), void (*addReexport)(const uint16_t* start, const uint16_t* end)); +void (*addRequire)(const uint16_t*, const uint16_t*) = &_addRequire; +bool parseCJS ( + uint16_t* source, + uint32_t sourceLen, + void (*addExport)(const uint16_t* start, const uint16_t* end), + void (*addReexport)(const uint16_t* start, const uint16_t* end), + void (*addRequire)(const uint16_t* start, const uint16_t* end) +); enum RequireType { Import, diff --git a/include/cjs-module-lexer.h b/include/cjs-module-lexer.h index 0bb8912..6056f20 100755 --- a/include/cjs-module-lexer.h +++ b/include/cjs-module-lexer.h @@ -27,7 +27,7 @@ typedef struct StarExportBinding StarExportBinding; void bail (uint32_t err); -bool parseCJS (uint16_t* source, uint32_t sourceLen, void (*addExport)(const uint16_t*, const uint16_t*), void (*addReexport)(const uint16_t*, const uint16_t*)); +bool parseCJS (uint16_t* source, uint32_t sourceLen, void (*addExport)(const uint16_t*, const uint16_t*), void (*addReexport)(const uint16_t*, const uint16_t*), void (*addRequire)(const uint16_t*, const uint16_t*)); enum RequireType { Import, diff --git a/lexer.js b/lexer.js index 7f7c971..489202b 100755 --- a/lexer.js +++ b/lexer.js @@ -12,7 +12,8 @@ let openTokenDepth, lastStarExportSpecifier, lastExportsAssignSpecifier, _exports, - reexports; + reexports, + requires; function resetState () { openTokenDepth = 0; @@ -30,6 +31,7 @@ function resetState () { _exports = new Set(); reexports = new Set(); + requires = []; } // RequireType @@ -51,7 +53,7 @@ module.exports = function parseCJS (source, name = '@') { } if (lastExportsAssignSpecifier) reexports.add(lastExportsAssignSpecifier); - const result = { exports: [..._exports], reexports: [...reexports] }; + const result = { exports: [..._exports], reexports: [...reexports], requires }; resetState(); return result; } @@ -654,6 +656,7 @@ function tryParseRequire (requireType) { const reexportEnd = pos++; ch = commentWhitespace(); if (ch === 41/*)*/) { + requires.push({ s: reexportStart - 1, e: reexportEnd + 1 }); switch (requireType) { case ExportAssign: lastExportsAssignSpecifier = source.slice(reexportStart, reexportEnd); @@ -672,6 +675,7 @@ function tryParseRequire (requireType) { const reexportEnd = pos++; ch = commentWhitespace(); if (ch === 41/*)*/) { + requires.push({ s: reexportStart - 1, e: reexportEnd + 1 }); switch (requireType) { case ExportAssign: lastExportsAssignSpecifier = source.slice(reexportStart, reexportEnd); diff --git a/src/lexer.c b/src/lexer.c index b3c7c99..974c0f5 100755 --- a/src/lexer.c +++ b/src/lexer.c @@ -43,15 +43,18 @@ const StarExportBinding* STAR_EXPORT_STACK_END = &starExportStack_[MAX_STAR_EXPO void (*addExport)(const uint16_t*, const uint16_t*); void (*addReexport)(const uint16_t*, const uint16_t*); +void (*addRequire)(const uint16_t*, const uint16_t*); // Note: parsing is based on the _assumption_ that the source is already valid -bool parseCJS (uint16_t* _source, uint32_t _sourceLen, void (*_addExport)(const uint16_t*, const uint16_t*), void (*_addReexport)(const uint16_t*, const uint16_t*)) { +bool parseCJS (uint16_t* _source, uint32_t _sourceLen, void (*_addExport)(const uint16_t*, const uint16_t*), void (*_addReexport)(const uint16_t*, const uint16_t*), void (*_addRequire)(const uint16_t*, const uint16_t*)) { source = _source; sourceLen = _sourceLen; if (_addExport) addExport = _addExport; if (_addReexport) addReexport = _addReexport; + if (_addRequire) + addRequire = _addRequire; templateStackDepth = 0; openTokenDepth = 0; @@ -676,6 +679,7 @@ bool tryParseRequire (enum RequireType requireType) { uint16_t* reexportEnd = pos++; ch = commentWhitespace(); if (ch == ')') { + addRequire(reexportStart - 1, reexportEnd + 1); switch (requireType) { case ExportStar: addReexport(reexportStart, reexportEnd); @@ -696,6 +700,7 @@ bool tryParseRequire (enum RequireType requireType) { uint16_t* reexportEnd = pos++; ch = commentWhitespace(); if (ch == ')') { + addRequire(reexportStart - 1, reexportEnd + 1); switch (requireType) { case ExportStar: addReexport(reexportStart, reexportEnd); diff --git a/src/lexer.js b/src/lexer.js index f5e01c6..e11918a 100755 --- a/src/lexer.js +++ b/src/lexer.js @@ -17,7 +17,7 @@ export function parse (source, name = '@') { if (!wasm.parseCJS(addr, source.length, 0, 0)) throw Object.assign(new Error(`Parse error ${name}${wasm.e()}:${source.slice(0, wasm.e()).split('\n').length}:${wasm.e() - source.lastIndexOf('\n', wasm.e() - 1)}`), { idx: wasm.e() }); - let exports = new Set(), reexports = new Set(); + let exports = new Set(), reexports = new Set(), requires = []; while (wasm.rre()) reexports.add(source.slice(wasm.res(), wasm.ree())); while (wasm.re()) { @@ -25,8 +25,10 @@ export function parse (source, name = '@') { if (!strictReserved.has(exptStr)) exports.add(exptStr); } + while (wasm.rrq()) + requires.push({ s: wasm.rqs(), e: wasm.rqe() }); - return { exports: [...exports], reexports: [...reexports] }; + return { exports: [...exports], reexports: [...reexports], requires }; } function copy (src, outBuf16) { diff --git a/test/_unit.js b/test/_unit.js index 5b8d8f8..d376ac3 100755 --- a/test/_unit.js +++ b/test/_unit.js @@ -17,7 +17,7 @@ suite('Lexer', () => { beforeEach(async () => await loadParser()); test('TypeScript reexports', () => { - var { exports, reexports } = parse(` + var { exports, reexports, requires } = parse(` "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; @@ -35,6 +35,12 @@ suite('Lexer', () => { assert.equal(reexports[1], 'external2'); assert.equal(reexports[2], 'external3'); assert.equal(reexports[3], 'external4'); + + assert.equal(requires.length, 4); + assert.deepEqual(requires[0], { s: 228, e: 239 }); + assert.deepEqual(requires[1], { s: 272, e: 283 }); + assert.deepEqual(requires[2], { s: 314, e: 325 }); + assert.deepEqual(requires[3], { s: 363, e: 374 }); }); test('Rollup Babel reexports', () => { @@ -427,6 +433,25 @@ suite('Lexer', () => { assert.equal(reexports[0], './another'); }); + + test('Requires', () => { + const source = ` + const a = require("module/a"); + const b = require("./module-b.js"); + `; + const { exports, reexports, requires } = parse(source); + assert.equal(requires.length, 2); + + assert.deepEqual(requires[0], { s: 25, e: 35 }); + assert.equal(source.slice(requires[0].s, requires[0].e), `"module/a"`); + + assert.deepEqual(requires[1], { s: 62, e: 77 }); + assert.equal(source.slice(requires[1].s, requires[1].e), `"./module-b.js"`); + + assert.equal(exports.length, 0); + assert.equal(reexports.length, 0); + }); + test('Single parse cases', () => { parse(`'asdf'`); parse(`/asdf/`);