[esm-integration] Add a new core test mode and either fix or disable all test.

This is fairly larger change which can likely to split up before landing.

Author: sbc100

.circleci/config.yml

@@ -659,6 +659,21 @@ jobs:
     steps:
       - run-tests-linux:
           test_targets: "instance"
+  test-esm-integration:
+    # We don't use `bionic` here since its too old to run recent node versions:
+    # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found`
+    executor: linux-python
+    steps:
+      - prepare-for-tests
+      # The linux-python image uses /home/circleci rather than /root and jsvu
+      # hardcodes /root into its launcher scripts so we need to reinstall v8.
+      - run: rm -rf $HOME/.jsvu
+      - install-v8
+      - install-node-canary
+      - run-tests:
+          title: "esm-integration"
+          test_targets: "esm-integration"
+      - upload-test-results
   test-wasm2js1:
     environment:
       EMTEST_SKIP_NODE_CANARY: "1"
@@ -667,7 +682,7 @@ jobs:
       - run-tests-linux:
           test_targets: "wasm2js1"
   test-wasm64:
-    # We don't use `bionic` here since its tool old to run recent node versions:
+    # We don't use `bionic` here since its too old to run recent node versions:
     # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found`
     executor: linux-python
     steps:
@@ -811,10 +826,6 @@ jobs:
             core0.test_pthread_join_and_asyncify
             core0.test_async_ccall_promise_jspi*
             core0.test_cubescript_jspi
-            esm_integration.test_fs_js_api*
-            esm_integration.test_inlinejs3
-            esm_integration.test_embind_val_basics
-            esm_integration.test_undefined_main
             "
       # Run some basic tests with the minimum version of node that we currently
       # support in the generated code.
@@ -1142,6 +1153,9 @@ workflows:
       - test-modularize-instance:
           requires:
             - build-linux
+      - test-esm-integration:
+          requires:
+            - build-linux
       - test-browser-chrome
       - test-browser-chrome-2gb:
           requires:

site/source/docs/compiling/Modularized-Output.rst

@@ -138,6 +138,7 @@ fix in future releses.  Current limitations include:
 * The output of file_packager is not compatible so :ref:`emcc-preload-file` and
   :ref:`emcc-embed-file` do not work.
 
+
 Source Phase Imports (experimental)
 ===================================
 
@@ -167,5 +168,14 @@ This setting implicitly enables :ref:`export_es6` and sets :ref:`MODULARIZE` to
 ``instance``.  Because of this all the same limitations mentioned above for
 ``-sMODULARIZE=intance`` apply.
 
+Some additional limitations are:
+
+- ``-pthread`` / :ref:`wasm_workers` are not yet supported.
+
+- Setting :ref:`wasm` to ``0`` is not supported.
+
+- Setting :ref:`wasm_async_compilation` to ``0`` is not supported.
+
+
 .. _Source phase imports: https://github.com/tc39/proposal-source-phase-imports
 .. _Wasm ESM integration: https://github.com/WebAssembly/esm-integration

test/common.py

@@ -522,7 +522,7 @@ def metafunc(self, with_minimal_runtime, *args, **kwargs):
       print('parameterize:minimal_runtime=%s' % with_minimal_runtime)
     assert self.get_setting('MINIMAL_RUNTIME') is None
     if with_minimal_runtime:
-      if self.get_setting('MODULARIZE') == 'instance':
+      if self.get_setting('MODULARIZE') == 'instance' or self.get_setting('WASM_ESM_INTEGRATION'):
         self.skipTest('MODULARIZE=instance is not compatible with MINIMAL_RUNTIME')
       self.set_setting('MINIMAL_RUNTIME', 1)
       # This extra helper code is needed to cleanly handle calls to exit() which throw
@@ -604,6 +604,7 @@ def can_do_standalone(self, impure=False):
   return self.is_wasm() and \
       self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 and \
       not self.get_setting('MINIMAL_RUNTIME') and \
+      not self.get_setting('WASM_ESM_INTEGRATION') and \
       not self.get_setting('SAFE_HEAP') and \
       not any(a.startswith('-fsanitize=') for a in self.emcc_args)
 
@@ -1138,6 +1139,8 @@ def require_wasm2js(self):
       self.skipTest('wasm2js is not compatible with MEMORY64')
     if self.is_2gb() or self.is_4gb():
       self.skipTest('wasm2js does not support over 2gb of memory')
+    if self.get_setting('WASM_ESM_INTEGRATION'):
+      self.skipTest('wasm2js is not compatible with WASM_ESM_INTEGRATION')
 
   def setup_nodefs_test(self):
     self.require_node()
@@ -1160,6 +1163,8 @@ def setup_node_pthreads(self):
     self.emcc_args += ['-Wno-pthreads-mem-growth', '-pthread']
     if self.get_setting('MINIMAL_RUNTIME'):
       self.skipTest('node pthreads not yet supported with MINIMAL_RUNTIME')
+    if self.get_setting('WASM_ESM_INTEGRATION'):
+      self.skipTest('pthreads not yet supported with WASM_ESM_INTEGRATION')
     nodejs = self.get_nodejs()
     self.js_engines = [nodejs]
     self.node_args += shared.node_pthread_flags(nodejs)

test/core/test_demangle_stacks.cpp

@@ -8,6 +8,8 @@
 
 #include <emscripten.h>
 
+EM_JS_DEPS(deps, "$jsStackTrace");
+
 namespace NameSpace {
 class Class {
  public:

test/other/test_unoptimized_code_size.js.size

@@ -1 +1 @@
-53704
+53737

test/other/test_unoptimized_code_size_no_asserts.js.size

@@ -1 +1 @@
-26959
+26992

test/other/test_unoptimized_code_size_strict.js.size

@@ -1 +1 @@
-51754
+51787

test/test_core.py

@@ -67,6 +67,22 @@ def decorated(self, *args, **kwargs):
   return decorator
 
 
+def no_esm_integration(note):
+  assert not callable(note)
+
+  def decorator(f):
+    assert callable(f)
+
+    @wraps(f)
+    def decorated(self, *args, **kwargs):
+      if self.get_setting('WASM_ESM_INTEGRATION'):
+        self.skipTest(note)
+      f(self, *args, **kwargs)
+    return decorated
+
+  return decorator
+
+
 def wasm_simd(f):
   assert callable(f)
 
@@ -189,6 +205,8 @@ def with_asyncify_and_jspi(f):
 
   @wraps(f)
   def metafunc(self, jspi, *args, **kwargs):
+    if self.get_setting('WASM_ESM_INTEGRATION'):
+      self.skipTest('WASM_ESM_INTEGRATION is not compatible with ASYNCIFY')
     if jspi:
       self.set_setting('ASYNCIFY', 2)
       self.require_jspi()
@@ -208,6 +226,8 @@ def also_with_asyncify_and_jspi(f):
 
   @wraps(f)
   def metafunc(self, asyncify, *args, **kwargs):
+    if asyncify and self.get_setting('WASM_ESM_INTEGRATION'):
+      self.skipTest('WASM_ESM_INTEGRATION is not compatible with ASYNCIFY')
     if asyncify == 2:
       self.set_setting('ASYNCIFY', 2)
       self.require_jspi()
@@ -424,6 +444,7 @@ def get_bullet_library(self, use_cmake):
   def test_hello_world(self):
     self.do_core_test('test_hello_world.c')
 
+  @no_esm_integration('WASM_ASYNC_COMPILATION=0')
   def test_wasm_synchronous_compilation(self):
     if self.get_setting('MODULARIZE') != 'instance':
       self.set_setting('STRICT_JS')
@@ -914,6 +935,7 @@ def test_longjmp(self):
     self.do_core_test('test_longjmp.c')
 
   @no_sanitize('sanitizers do not support WASM_WORKERS')
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS')
   def test_longjmp_wasm_workers(self):
     self.do_core_test('test_longjmp.c', emcc_args=['-sWASM_WORKERS'])
 
@@ -1995,7 +2017,11 @@ def test_em_js(self, args, force_c):
       self.setup_node_pthreads()
 
     self.do_core_test('test_em_js.cpp', force_c=force_c)
-    self.assertContained('no args returning int', read_file(self.output_name('test_em_js')))
+    if self.get_setting('WASM_ESM_INTEGRATION'):
+      js_out = 'test_em_js.support.mjs'
+    else:
+      js_out = self.output_name('test_em_js')
+    self.assertContained('no args returning int', read_file(js_out))
 
   @no_wasm2js('test depends on WASM_BIGINT which is not compatible with wasm2js')
   def test_em_js_i64(self):
@@ -2173,6 +2199,7 @@ def test_nothrow_new(self, args):
   @no_lsan('LSan alters the memory size')
   @no_4gb('depends on memory size')
   @no_2gb('depends on memory size')
+  @no_esm_integration('external wasmMemory')
   def test_module_wasm_memory(self):
     self.emcc_args += ['--pre-js', test_file('core/test_module_wasm_memory.js')]
     self.set_setting('IMPORTED_MEMORY')
@@ -6982,7 +7009,6 @@ def test_EXPORTED_RUNTIME_METHODS(self):
 
   @also_with_minimal_runtime
   @no_modularize_instance('uses dynCallLegacy')
-  @no_wasm64('not compatible with MEMORY64')
   def test_dyncall_specific(self):
     if self.get_setting('WASM_BIGINT') != 0 and not self.is_wasm2js():
       # define DYNCALLS because this test does test calling them directly, and
@@ -7446,6 +7472,7 @@ def test_embind_negative_constants(self):
     self.do_run_in_out_file_test('embind/test_negative_constants.cpp', emcc_args=['-lembind'])
 
   @also_with_wasm_bigint
+  @no_esm_integration('embind is not compatible with WASM_ESM_INTEGRATION')
   def test_embind_unsigned(self):
     self.do_run_in_out_file_test('embind/test_unsigned.cpp', emcc_args=['-lembind'])
 
@@ -7621,6 +7648,7 @@ def test_embind_no_rtti_followed_by_rtti(self):
     self.do_run(src, '418\ndotest returned: 42\n')
 
   @no_sanitize('sanitizers do not support WASM_WORKERS')
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS')
   def test_embind_wasm_workers(self):
     self.do_run_in_out_file_test('embind/test_embind_wasm_workers.cpp', emcc_args=['-lembind', '-sWASM_WORKERS'])
 
@@ -7796,6 +7824,7 @@ def test_embind_dylink_visibility_hidden(self):
     self.do_runf('main.cpp', 'done\n', emcc_args=['--bind'])
 
   @no_wasm2js('TODO: source maps in wasm2js')
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with dwarf output')
   def test_dwarf(self):
     self.emcc_args.append('-g')
 
@@ -8238,6 +8267,7 @@ def test_asyncify_lists(self, args, should_pass, response=None):
         binary = read_binary(filename)
         self.assertFalse(b'main' in binary)
 
+  @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with ASYNCIFY')
   @parameterized({
     'normal': ([], True),
     'ignoreindirect': (['-sASYNCIFY_IGNORE_INDIRECT'], False),
@@ -8629,7 +8659,10 @@ def test_environment(self):
 
     def test(assert_returncode=0):
       self.do_core_test('test_hello_world.c', assert_returncode=assert_returncode)
-      js = read_file(self.output_name('test_hello_world'))
+      if self.get_setting('WASM_ESM_INTEGRATION'):
+        js = read_file(self.output_name('test_hello_world.support'))
+      else:
+        js = read_file(self.output_name('test_hello_world'))
       assert ('require(' in js) == ('node' in self.get_setting('ENVIRONMENT')), 'we should have require() calls only if node js specified'
 
     for engine in config.JS_ENGINES:
@@ -8754,11 +8787,13 @@ def test_minimal_runtime_global_initializer(self):
     self.do_runf('test_global_initializer.cpp', 't1 > t0: 1')
 
   @no_wasm2js('wasm2js does not support PROXY_TO_PTHREAD (custom section support)')
+  @no_esm_integration('USE_OFFSET_CONVERTER')
   def test_return_address(self):
     self.set_setting('USE_OFFSET_CONVERTER')
     self.do_runf('core/test_return_address.c', 'passed')
 
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   @no_asan('-fsanitize-minimal-runtime cannot be used with ASan')
   @no_lsan('-fsanitize-minimal-runtime cannot be used with LSan')
   def test_ubsan_minimal_too_many_errors(self):
@@ -8768,6 +8803,7 @@ def test_ubsan_minimal_too_many_errors(self):
                  regex=True)
 
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   @no_asan('-fsanitize-minimal-runtime cannot be used with ASan')
   @no_lsan('-fsanitize-minimal-runtime cannot be used with LSan')
   def test_ubsan_minimal_errors_same_place(self):
@@ -8782,6 +8818,7 @@ def test_ubsan_minimal_errors_same_place(self):
     'fsanitize_overflow': (['-fsanitize=signed-integer-overflow'],),
   })
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_ubsan_full_overflow(self, args):
     self.emcc_args += args
     self.do_runf(
@@ -8797,6 +8834,7 @@ def test_ubsan_full_overflow(self, args):
     'fsanitize_return': (['-fsanitize=return'],),
   })
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_ubsan_full_no_return(self, args):
     self.emcc_args += ['-Wno-return-type'] + args
     self.do_runf('core/test_ubsan_full_no_return.cpp',
@@ -8808,6 +8846,7 @@ def test_ubsan_full_no_return(self, args):
     'fsanitize_shift': (['-fsanitize=shift'],),
   })
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_ubsan_full_left_shift(self, args):
     self.emcc_args += args
     self.do_runf(
@@ -8824,6 +8863,7 @@ def test_ubsan_full_left_shift(self, args):
     'dylink': (['-fsanitize=null', '-sMAIN_MODULE=2'],),
   })
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_ubsan_full_null_ref(self, args):
     if '-sMAIN_MODULE=2' in args:
       self.check_dylink()
@@ -8840,6 +8880,7 @@ def test_ubsan_full_null_ref(self, args):
       ])
 
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_sanitize_vptr(self):
     self.do_runf(
       'core/test_sanitize_vptr.cpp',
@@ -8862,6 +8903,7 @@ def test_sanitize_vptr(self):
     ]),
   })
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_ubsan_full_stack_trace(self, g_flag, expected_output):
     if g_flag == '-gsource-map':
       if self.is_wasm2js():
@@ -8876,6 +8918,7 @@ def test_ubsan_full_stack_trace(self, g_flag, expected_output):
                  assert_all=True, expected_output=expected_output)
 
   @no_wasm2js('TODO: sanitizers in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_ubsan_typeinfo_eq(self):
     # https://github.com/emscripten-core/emscripten/issues/13330
     src = r'''
@@ -8895,6 +8938,7 @@ def test_template_class_deduction(self):
     self.do_core_test('test_template_class_deduction.cpp')
 
   @no_wasm2js('TODO: ASAN in wasm2js')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   @no_safe_heap('asan does not work with SAFE_HEAP')
   @no_wasm64('TODO: ASAN in memory64')
   @no_2gb('asan doesnt support GLOBAL_BASE')
@@ -8915,6 +8959,7 @@ def test_asan_no_error(self, name):
   @no_safe_heap('asan does not work with SAFE_HEAP')
   @no_wasm64('TODO: ASAN in memory64')
   @no_2gb('asan doesnt support GLOBAL_BASE')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   @parameterized({
     'use_after_free_c': ('test_asan_use_after_free.c', [
       'AddressSanitizer: heap-use-after-free on address',
@@ -8988,6 +9033,7 @@ def test_asan(self, name, expected_output, cflags=None):
   @no_wasm2js('TODO: ASAN in wasm2js')
   @no_wasm64('TODO: ASAN in memory64')
   @no_2gb('asan doesnt support GLOBAL_BASE')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_asan_js_stack_op(self):
     self.emcc_args.append('-fsanitize=address')
     self.set_setting('ALLOW_MEMORY_GROWTH')
@@ -8999,6 +9045,7 @@ def test_asan_js_stack_op(self):
   @no_wasm2js('TODO: ASAN in wasm2js')
   @no_wasm64('TODO: ASAN in memory64')
   @no_2gb('asan doesnt support GLOBAL_BASE')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_asan_api(self):
     self.emcc_args.append('-fsanitize=address')
     self.set_setting('INITIAL_MEMORY', '300mb')
@@ -9008,6 +9055,7 @@ def test_asan_api(self):
   @no_wasm2js('TODO: ASAN in wasm2js')
   @no_wasm64('TODO: ASAN in memory64')
   @no_2gb('asan doesnt support GLOBAL_BASE')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_asan_modularized_with_closure(self):
     # the bug is that createModule() returns undefined, instead of the
     # proper Promise object.
@@ -9022,6 +9070,7 @@ def test_asan_modularized_with_closure(self):
 
   @no_asan('SAFE_HEAP cannot be used with ASan')
   @no_2gb('asan doesnt support GLOBAL_BASE')
+  @no_esm_integration('sanitizers do not support WASM_ESM_INTEGRATION')
   def test_safe_heap_user_js(self):
     self.set_setting('SAFE_HEAP')
     self.do_runf('core/test_safe_heap_user_js.c',
@@ -9413,6 +9462,7 @@ def test_Module_dynamicLibraries(self, args):
 
   # Tests the emscripten_get_exported_function() API.
   @also_with_minimal_runtime
+  @no_esm_integration('depends on wasmExports')
   def test_get_exported_function(self):
     self.set_setting('ALLOW_TABLE_GROWTH')
     self.emcc_args += ['-lexports.js']