diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a6ba54e9a38e..3216224662940 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -659,6 +659,21 @@ jobs: steps: - run-tests-linux: test_targets: "instance" + test-esm-integration: + # We don't use `bionic` here since its too old to run recent node versions: + # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found` + executor: linux-python + steps: + - prepare-for-tests + # The linux-python image uses /home/circleci rather than /root and jsvu + # hardcodes /root into its launcher scripts so we need to reinstall v8. + - run: rm -rf $HOME/.jsvu + - install-v8 + - install-node-canary + - run-tests: + title: "esm-integration" + test_targets: "esm-integration" + - upload-test-results test-wasm2js1: environment: EMTEST_SKIP_NODE_CANARY: "1" @@ -667,7 +682,7 @@ jobs: - run-tests-linux: test_targets: "wasm2js1" test-wasm64: - # We don't use `bionic` here since its tool old to run recent node versions: + # We don't use `bionic` here since its too old to run recent node versions: # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found` executor: linux-python steps: @@ -811,10 +826,6 @@ jobs: core0.test_pthread_join_and_asyncify core0.test_async_ccall_promise_jspi* core0.test_cubescript_jspi - esm_integration.test_fs_js_api* - esm_integration.test_inlinejs3 - esm_integration.test_embind_val_basics - esm_integration.test_undefined_main " # Run some basic tests with the minimum version of node that we currently # support in the generated code. @@ -1142,6 +1153,9 @@ workflows: - test-modularize-instance: requires: - build-linux + - test-esm-integration: + requires: + - build-linux - test-browser-chrome - test-browser-chrome-2gb: requires: diff --git a/site/source/docs/compiling/Modularized-Output.rst b/site/source/docs/compiling/Modularized-Output.rst index b726ff1248027..5701c8fadd6f0 100644 --- a/site/source/docs/compiling/Modularized-Output.rst +++ b/site/source/docs/compiling/Modularized-Output.rst @@ -138,6 +138,7 @@ fix in future releses. Current limitations include: * The output of file_packager is not compatible so :ref:`emcc-preload-file` and :ref:`emcc-embed-file` do not work. + Source Phase Imports (experimental) =================================== @@ -167,5 +168,14 @@ This setting implicitly enables :ref:`export_es6` and sets :ref:`MODULARIZE` to ``instance``. Because of this all the same limitations mentioned above for ``-sMODULARIZE=intance`` apply. +Some additional limitations are: + +- ``-pthread`` / :ref:`wasm_workers` are not yet supported. + +- Setting :ref:`wasm` to ``0`` is not supported. + +- Setting :ref:`wasm_async_compilation` to ``0`` is not supported. + + .. _Source phase imports: https://github.com/tc39/proposal-source-phase-imports .. _Wasm ESM integration: https://github.com/WebAssembly/esm-integration diff --git a/test/common.py b/test/common.py index d99a5c76fefd4..d26f1f3ee8f29 100644 --- a/test/common.py +++ b/test/common.py @@ -522,7 +522,7 @@ def metafunc(self, with_minimal_runtime, *args, **kwargs): print('parameterize:minimal_runtime=%s' % with_minimal_runtime) assert self.get_setting('MINIMAL_RUNTIME') is None if with_minimal_runtime: - if self.get_setting('MODULARIZE') == 'instance': + if self.get_setting('MODULARIZE') == 'instance' or self.get_setting('WASM_ESM_INTEGRATION'): self.skipTest('MODULARIZE=instance is not compatible with MINIMAL_RUNTIME') self.set_setting('MINIMAL_RUNTIME', 1) # This extra helper code is needed to cleanly handle calls to exit() which throw @@ -604,6 +604,7 @@ def can_do_standalone(self, impure=False): return self.is_wasm() and \ self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 and \ not self.get_setting('MINIMAL_RUNTIME') and \ + not self.get_setting('WASM_ESM_INTEGRATION') and \ not self.get_setting('SAFE_HEAP') and \ not any(a.startswith('-fsanitize=') for a in self.emcc_args) @@ -1138,6 +1139,8 @@ def require_wasm2js(self): self.skipTest('wasm2js is not compatible with MEMORY64') if self.is_2gb() or self.is_4gb(): self.skipTest('wasm2js does not support over 2gb of memory') + if self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest('wasm2js is not compatible with WASM_ESM_INTEGRATION') def setup_nodefs_test(self): self.require_node() @@ -1160,6 +1163,8 @@ def setup_node_pthreads(self): self.emcc_args += ['-Wno-pthreads-mem-growth', '-pthread'] if self.get_setting('MINIMAL_RUNTIME'): self.skipTest('node pthreads not yet supported with MINIMAL_RUNTIME') + if self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest('pthreads not yet supported with WASM_ESM_INTEGRATION') nodejs = self.get_nodejs() self.js_engines = [nodejs] self.node_args += shared.node_pthread_flags(nodejs) diff --git a/test/core/test_demangle_stacks.cpp b/test/core/test_demangle_stacks.cpp index 25a78b86552a4..fb71a46541060 100644 --- a/test/core/test_demangle_stacks.cpp +++ b/test/core/test_demangle_stacks.cpp @@ -8,6 +8,8 @@ #include +EM_JS_DEPS(deps, "$jsStackTrace"); + namespace NameSpace { class Class { public: diff --git a/test/other/test_unoptimized_code_size.js.size b/test/other/test_unoptimized_code_size.js.size index 54d8bb55053f8..ddfb6f1fe7e88 100644 --- a/test/other/test_unoptimized_code_size.js.size +++ b/test/other/test_unoptimized_code_size.js.size @@ -1 +1 @@ -53827 +53860 diff --git a/test/other/test_unoptimized_code_size_no_asserts.js.size b/test/other/test_unoptimized_code_size_no_asserts.js.size index bf6484c5363f9..8203b5feb27cd 100644 --- a/test/other/test_unoptimized_code_size_no_asserts.js.size +++ b/test/other/test_unoptimized_code_size_no_asserts.js.size @@ -1 +1 @@ -27082 +27115 diff --git a/test/other/test_unoptimized_code_size_strict.js.size b/test/other/test_unoptimized_code_size_strict.js.size index 72ccf968ca7f2..94d4d2fd8e440 100644 --- a/test/other/test_unoptimized_code_size_strict.js.size +++ b/test/other/test_unoptimized_code_size_strict.js.size @@ -1 +1 @@ -51877 +51910 diff --git a/test/test_core.py b/test/test_core.py index f2a0a973d7d0a..ada62c8654f47 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -67,6 +67,22 @@ def decorated(self, *args, **kwargs): return decorator +def no_esm_integration(note): + assert not callable(note) + + def decorator(f): + assert callable(f) + + @wraps(f) + def decorated(self, *args, **kwargs): + if self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest(note) + f(self, *args, **kwargs) + return decorated + + return decorator + + def wasm_simd(f): assert callable(f) @@ -189,6 +205,8 @@ def with_asyncify_and_jspi(f): @wraps(f) def metafunc(self, jspi, *args, **kwargs): + if self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest('WASM_ESM_INTEGRATION is not compatible with ASYNCIFY') if jspi: self.set_setting('ASYNCIFY', 2) self.require_jspi() @@ -208,6 +226,8 @@ def also_with_asyncify_and_jspi(f): @wraps(f) def metafunc(self, asyncify, *args, **kwargs): + if asyncify and self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest('WASM_ESM_INTEGRATION is not compatible with ASYNCIFY') if asyncify == 2: self.set_setting('ASYNCIFY', 2) self.require_jspi() @@ -437,6 +457,7 @@ def get_bullet_library(self, use_cmake): def test_hello_world(self): self.do_core_test('test_hello_world.c') + @no_esm_integration('WASM_ASYNC_COMPILATION=0') def test_wasm_synchronous_compilation(self): if self.get_setting('MODULARIZE') != 'instance': self.set_setting('STRICT_JS') @@ -927,6 +948,7 @@ def test_longjmp(self): self.do_core_test('test_longjmp.c') @no_sanitize('sanitizers do not support WASM_WORKERS') + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS') def test_longjmp_wasm_workers(self): self.do_core_test('test_longjmp.c', emcc_args=['-sWASM_WORKERS']) @@ -2008,7 +2030,11 @@ def test_em_js(self, args, force_c): self.setup_node_pthreads() self.do_core_test('test_em_js.cpp', force_c=force_c) - self.assertContained('no args returning int', read_file(self.output_name('test_em_js'))) + if self.get_setting('WASM_ESM_INTEGRATION'): + js_out = 'test_em_js.support.mjs' + else: + js_out = self.output_name('test_em_js') + self.assertContained('no args returning int', read_file(js_out)) @no_wasm2js('test depends on WASM_BIGINT which is not compatible with wasm2js') def test_em_js_i64(self): @@ -2186,6 +2212,7 @@ def test_nothrow_new(self, args): @no_lsan('LSan alters the memory size') @no_4gb('depends on memory size') @no_2gb('depends on memory size') + @no_esm_integration('external wasmMemory') def test_module_wasm_memory(self): self.emcc_args += ['--pre-js', test_file('core/test_module_wasm_memory.js')] self.set_setting('IMPORTED_MEMORY') @@ -6998,7 +7025,6 @@ def test_EXPORTED_RUNTIME_METHODS(self): @also_with_minimal_runtime @no_modularize_instance('uses dynCallLegacy') - @no_wasm64('not compatible with MEMORY64') def test_dyncall_specific(self): if self.get_setting('WASM_BIGINT') != 0 and not self.is_wasm2js(): # define DYNCALLS because this test does test calling them directly, and @@ -7462,6 +7488,7 @@ def test_embind_negative_constants(self): self.do_run_in_out_file_test('embind/test_negative_constants.cpp', emcc_args=['-lembind']) @also_with_wasm_bigint + @no_esm_integration('embind is not compatible with WASM_ESM_INTEGRATION') def test_embind_unsigned(self): self.do_run_in_out_file_test('embind/test_unsigned.cpp', emcc_args=['-lembind']) @@ -7637,6 +7664,7 @@ def test_embind_no_rtti_followed_by_rtti(self): self.do_run(src, '418\ndotest returned: 42\n') @no_sanitize('sanitizers do not support WASM_WORKERS') + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS') def test_embind_wasm_workers(self): self.do_run_in_out_file_test('embind/test_embind_wasm_workers.cpp', emcc_args=['-lembind', '-sWASM_WORKERS']) @@ -7812,6 +7840,7 @@ def test_embind_dylink_visibility_hidden(self): self.do_runf('main.cpp', 'done\n', emcc_args=['--bind']) @no_wasm2js('TODO: source maps in wasm2js') + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with dwarf output') def test_dwarf(self): self.emcc_args.append('-g') @@ -8254,6 +8283,7 @@ def test_asyncify_lists(self, args, should_pass, response=None): binary = read_binary(filename) self.assertFalse(b'main' in binary) + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with ASYNCIFY') @parameterized({ 'normal': ([], True), 'ignoreindirect': (['-sASYNCIFY_IGNORE_INDIRECT'], False), @@ -8645,7 +8675,10 @@ def test_environment(self): def test(assert_returncode=0): self.do_core_test('test_hello_world.c', assert_returncode=assert_returncode) - js = read_file(self.output_name('test_hello_world')) + if self.get_setting('WASM_ESM_INTEGRATION'): + js = read_file(self.output_name('test_hello_world.support')) + else: + js = read_file(self.output_name('test_hello_world')) assert ('require(' in js) == ('node' in self.get_setting('ENVIRONMENT')), 'we should have require() calls only if node js specified' for engine in config.JS_ENGINES: @@ -8770,11 +8803,13 @@ def test_minimal_runtime_global_initializer(self): self.do_runf('test_global_initializer.cpp', 't1 > t0: 1') @no_wasm2js('wasm2js does not support PROXY_TO_PTHREAD (custom section support)') + @no_esm_integration('USE_OFFSET_CONVERTER') def test_return_address(self): self.set_setting('USE_OFFSET_CONVERTER') self.do_runf('core/test_return_address.c', 'passed') @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') @no_asan('-fsanitize-minimal-runtime cannot be used with ASan') @no_lsan('-fsanitize-minimal-runtime cannot be used with LSan') def test_ubsan_minimal_too_many_errors(self): @@ -8784,6 +8819,7 @@ def test_ubsan_minimal_too_many_errors(self): regex=True) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') @no_asan('-fsanitize-minimal-runtime cannot be used with ASan') @no_lsan('-fsanitize-minimal-runtime cannot be used with LSan') def test_ubsan_minimal_errors_same_place(self): @@ -8798,6 +8834,7 @@ def test_ubsan_minimal_errors_same_place(self): 'fsanitize_overflow': (['-fsanitize=signed-integer-overflow'],), }) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_ubsan_full_overflow(self, args): self.emcc_args += args self.do_runf( @@ -8813,6 +8850,7 @@ def test_ubsan_full_overflow(self, args): 'fsanitize_return': (['-fsanitize=return'],), }) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_ubsan_full_no_return(self, args): self.emcc_args += ['-Wno-return-type'] + args self.do_runf('core/test_ubsan_full_no_return.cpp', @@ -8824,6 +8862,7 @@ def test_ubsan_full_no_return(self, args): 'fsanitize_shift': (['-fsanitize=shift'],), }) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_ubsan_full_left_shift(self, args): self.emcc_args += args self.do_runf( @@ -8840,6 +8879,7 @@ def test_ubsan_full_left_shift(self, args): 'dylink': (['-fsanitize=null', '-sMAIN_MODULE=2'],), }) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_ubsan_full_null_ref(self, args): if '-sMAIN_MODULE=2' in args: self.check_dylink() @@ -8856,6 +8896,7 @@ def test_ubsan_full_null_ref(self, args): ]) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_sanitize_vptr(self): self.do_runf( 'core/test_sanitize_vptr.cpp', @@ -8878,6 +8919,7 @@ def test_sanitize_vptr(self): ]), }) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_ubsan_full_stack_trace(self, g_flag, expected_output): if g_flag == '-gsource-map': if self.is_wasm2js(): @@ -8892,6 +8934,7 @@ def test_ubsan_full_stack_trace(self, g_flag, expected_output): assert_all=True, expected_output=expected_output) @no_wasm2js('TODO: sanitizers in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_ubsan_typeinfo_eq(self): # https://github.com/emscripten-core/emscripten/issues/13330 src = r''' @@ -8911,6 +8954,7 @@ def test_template_class_deduction(self): self.do_core_test('test_template_class_deduction.cpp') @no_wasm2js('TODO: ASAN in wasm2js') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') @no_safe_heap('asan does not work with SAFE_HEAP') @no_wasm64('TODO: ASAN in memory64') @no_2gb('asan doesnt support GLOBAL_BASE') @@ -8931,6 +8975,7 @@ def test_asan_no_error(self, name): @no_safe_heap('asan does not work with SAFE_HEAP') @no_wasm64('TODO: ASAN in memory64') @no_2gb('asan doesnt support GLOBAL_BASE') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') @parameterized({ 'use_after_free_c': ('test_asan_use_after_free.c', [ 'AddressSanitizer: heap-use-after-free on address', @@ -9004,6 +9049,7 @@ def test_asan(self, name, expected_output, cflags=None): @no_wasm2js('TODO: ASAN in wasm2js') @no_wasm64('TODO: ASAN in memory64') @no_2gb('asan doesnt support GLOBAL_BASE') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_asan_js_stack_op(self): self.emcc_args.append('-fsanitize=address') self.set_setting('ALLOW_MEMORY_GROWTH') @@ -9015,6 +9061,7 @@ def test_asan_js_stack_op(self): @no_wasm2js('TODO: ASAN in wasm2js') @no_wasm64('TODO: ASAN in memory64') @no_2gb('asan doesnt support GLOBAL_BASE') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_asan_api(self): self.emcc_args.append('-fsanitize=address') self.set_setting('INITIAL_MEMORY', '300mb') @@ -9024,6 +9071,7 @@ def test_asan_api(self): @no_wasm2js('TODO: ASAN in wasm2js') @no_wasm64('TODO: ASAN in memory64') @no_2gb('asan doesnt support GLOBAL_BASE') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_asan_modularized_with_closure(self): # the bug is that createModule() returns undefined, instead of the # proper Promise object. @@ -9038,6 +9086,7 @@ def test_asan_modularized_with_closure(self): @no_asan('SAFE_HEAP cannot be used with ASan') @no_2gb('asan doesnt support GLOBAL_BASE') + @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION') def test_safe_heap_user_js(self): self.set_setting('SAFE_HEAP') self.do_runf('core/test_safe_heap_user_js.c', @@ -9429,6 +9478,7 @@ def test_Module_dynamicLibraries(self, args): # Tests the emscripten_get_exported_function() API. @also_with_minimal_runtime + @no_esm_integration('depends on wasmExports') def test_get_exported_function(self): self.set_setting('ALLOW_TABLE_GROWTH') self.emcc_args += ['-lexports.js'] diff --git a/tools/emscripten.py b/tools/emscripten.py index e8f38891f456b..43a1997401703 100644 --- a/tools/emscripten.py +++ b/tools/emscripten.py @@ -961,23 +961,26 @@ def install_wrapper(sym): return wrappers -def create_receiving(function_exports): +def create_receiving(function_exports, tag_exports): + receiving = ['// Imports from the Wasm binary.'] + if settings.WASM_ESM_INTEGRATION: - exports = [f'{f} as {asmjs_mangle(f)}' for f in function_exports] + exports = tag_exports + list(function_exports.keys()) + exports = [f'{f} as {asmjs_mangle(f)}' for f in exports] if not settings.IMPORTED_MEMORY: exports.append('memory as wasmMemory') if not settings.RELOCATABLE: exports.append('__indirect_function_table as wasmTable') - exports = ',\n '.join(exports) - return f"import {{\n {exports}\n}} from './{settings.WASM_BINARY_FILE}';\n\n" + receiving.append('import {') + receiving.append(' ' + ',\n '.join(exports)) + receiving.append(f"}} from './{settings.WASM_BINARY_FILE}';") + return '\n'.join(receiving) + '\n\n' # When not declaring asm exports this section is empty and we instead programmatically export # symbols on the global object by calling exportWasmSymbols after initialization if not settings.DECLARE_ASM_MODULE_EXPORTS: return '' - receiving = [] - if settings.MINIMAL_RUNTIME: # Exports are assigned inside a function to variables # existing in top level JS scope, i.e. @@ -1008,13 +1011,13 @@ def create_receiving(function_exports): def create_module(metadata, function_exports, global_exports, tag_exports,library_symbols): module = [] - receiving = create_receiving(function_exports) + receiving = create_receiving(function_exports, tag_exports) + receiving += create_global_exports(global_exports) sending = create_sending(metadata, library_symbols) if settings.WASM_ESM_INTEGRATION: module.append(sending) else: - receiving += create_global_exports(global_exports) receiving += create_other_export_declarations(tag_exports) if settings.PTHREADS or settings.WASM_WORKERS or (settings.IMPORTED_MEMORY and settings.MODULARIZE == 'instance'): diff --git a/tools/link.py b/tools/link.py index d8700aee62acd..a2ca44b25c0ba 100644 --- a/tools/link.py +++ b/tools/link.py @@ -800,6 +800,16 @@ def phase_linker_setup(options, linker_args): # noqa: C901, PLR0912, PLR0915 exit_with_error('WASM_ESM_INTEGRATION requires MODULARIZE=instance') if settings.RELOCATABLE: exit_with_error('WASM_ESM_INTEGRATION is not compatible with dynamic linking') + if settings.WASM_WORKERS or settings.PTHREADS: + exit_with_error('WASM_ESM_INTEGRATION is not compatible with multi-threading') + if settings.USE_OFFSET_CONVERTER: + exit_with_error('WASM_ESM_INTEGRATION is not compatible with USE_OFFSET_CONVERTER') + if not settings.WASM_ASYNC_COMPILATION: + exit_with_error('WASM_ESM_INTEGRATION is not compatible with WASM_ASYNC_COMPILATION') + if not settings.WASM: + exit_with_error('WASM_ESM_INTEGRATION is not compatible with WASM2JS') + if settings.MAYBE_WASM2JS: + exit_with_error('WASM_ESM_INTEGRATION is not compatible with MAYBE_WASM2JS') if settings.MODULARIZE and settings.MODULARIZE not in [1, 'instance']: exit_with_error(f'Invalid setting "{settings.MODULARIZE}" for MODULARIZE.') @@ -2136,7 +2146,7 @@ def node_detection_code(): def create_esm_wrapper(wrapper_file, support_target, wasm_target): - js_exports = settings.EXPORTED_RUNTIME_METHODS + list(building.user_requested_exports) + js_exports = building.user_requested_exports.union(settings.EXPORTED_RUNTIME_METHODS) js_exports = ', '.join(sorted(js_exports)) wrapper = [] @@ -2171,7 +2181,10 @@ def create_esm_wrapper(wrapper_file, support_target, wasm_target): mod = mod.replace('(import "wasi_snapshot_preview1"', f'(import "{support_url}"') wasm_as = os.path.join(building.get_binaryen_bin(), 'wasm-as') - shared.check_call([wasm_as, '--all-features', '-o', wasm_target, '-'], input=mod) + cmd = [wasm_as, '--all-features', '-o', wasm_target, '-'] + if settings.EMIT_NAME_SECTION: + cmd.append('-g') + shared.check_call(cmd, input=mod) @ToolchainProfiler.profile_block('final emitting')