[wasm] Load assets earlier (#63623)

- allow Module.config to have assets with buffer instead of url
- change sample to show how emscripten callbacks are triggered
- pass config into onConfigLoaded and made it async
- split asset download from copying it into memory
- moved startup sequence to configure_emscripten_startup and documented it there
- diagnostic_tracing as CLI parameter for tests
- throttle parallel download of assets
- move tests which fail on V8 to browser or nodeJS scenarios

Author: pavelsavara

src/libraries/System.IO.Compression.ZipFile/tests/System.IO.Compression.ZipFile.Tests.csproj

@@ -4,6 +4,15 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser</TargetFrameworks>
   </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- This doesn't run on V8 shell because https://bugs.chromium.org/p/v8/issues/detail?id=12541 -->
+    <Scenario>WasmTestOnBrowser</Scenario>
+    <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
+    <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
+    <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="ZipFile.Create.cs" />
     <Compile Include="ZipFile.Extract.cs" />

src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj

@@ -3,6 +3,15 @@
     <TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- This doesn't run on V8 shell because https://bugs.chromium.org/p/v8/issues/detail?id=12541 -->
+    <Scenario>WasmTestOnBrowser</Scenario>
+    <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
+    <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
+    <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="CompressionStreamUnitTests.ZLib.cs" />
     <Compile Include="CompressionStreamUnitTests.Deflate.cs" />

src/libraries/System.Private.Xml/tests/Xslt/XslCompiledTransformApi/System.Xml.Xsl.XslCompiledTransformApi.Tests.csproj

@@ -2,6 +2,15 @@
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
   </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- This doesn't run on V8 shell because https://bugs.chromium.org/p/v8/issues/detail?id=12541 -->
+    <Scenario>WasmTestOnBrowser</Scenario>
+    <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
+    <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
+    <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="Errata4.cs" />
     <Compile Include="OutputSettings.cs" />

src/libraries/System.Private.Xml/tests/Xslt/XslTransformApi/System.Xml.Xsl.XslTransformApi.Tests.csproj

@@ -2,6 +2,15 @@
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
   </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- This doesn't run on V8 shell because https://bugs.chromium.org/p/v8/issues/detail?id=12541 -->
+    <Scenario>WasmTestOnBrowser</Scenario>
+    <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
+    <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
+    <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="cthread.cs" />
     <Compile Include="CThreads.cs" />

src/libraries/System.Private.Xml/tests/Xslt/XsltCompiler/XsltCompiler.Tests.csproj

@@ -2,6 +2,15 @@
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
   </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- This doesn't run on V8 shell because https://bugs.chromium.org/p/v8/issues/detail?id=12541 -->
+    <Scenario>WasmTestOnBrowser</Scenario>
+    <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
+    <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
+    <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="CommonScenarios\XsltcTestBasicFunctionality.cs" />
     <Compile Include="CommonScenarios\XsltcTestCaseBase.cs" />

src/libraries/sendtohelixhelp.proj

@@ -492,6 +492,9 @@
       <_WorkItem Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" />
 
       <_WorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnBrowser'" />
+      <_WorkItem Include="$(TestArchiveRoot)browserornodejs/**/*.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnBrowser'" />
+      <_WorkItem Include="$(TestArchiveRoot)browserornodejs/**/*.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnNodeJs'" />
+      <_WorkItem Include="$(TestArchiveRoot)nodejsonly/**/*.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnNodeJs'" />
 
       <HelixWorkItem Include="@(_WorkItem -> '$(WorkItemPrefix)%(FileName)')" Condition="'$(DefaultHelixWorkItems)' == 'true'">
         <PayloadArchive>%(Identity)</PayloadArchive>

src/libraries/tests.proj

@@ -325,9 +325,6 @@ Roslyn4.0.Tests.csproj" />
 
   <ItemGroup Condition="'$(TargetOS)' == 'Browser' and '$(BrowserHost)' == 'windows' and '$(RunDisabledWasmTestsOnWindows)' != 'true'">
     <!-- Issue: https://github.com/dotnet/runtime/issues/52138 -->
-    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Compression\tests\System.IO.Compression.Tests.csproj" />
-    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Compression.ZipFile\tests\System.IO.Compression.ZipFile.Tests.csproj" />
-    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Private.Xml\tests\Xslt\XslCompiledTransformApi\System.Xml.Xsl.XslCompiledTransformApi.Tests.csproj" />
     <ProjectExclusions Include="$(MonoProjectRoot)sample\wasm\console-v8-es6\Wasm.Console.V8.ES6.Sample.csproj" />
     <ProjectExclusions Include="$(MonoProjectRoot)sample\wasm\console-v8-cjs\Wasm.Console.V8.CJS.Sample.csproj" />
     <ProjectExclusions Include="$(MonoProjectRoot)sample\wasm\console-node-es6\Wasm.Console.Node.ES6.Sample.csproj" />

src/mono/sample/wasm/browser/main.js

@@ -17,10 +17,18 @@ async function loadRuntime() {
 async function main() {
     try {
         const createDotnetRuntime = await loadRuntime();
-        const { MONO, BINDING, Module, RuntimeBuildInfo } = await createDotnetRuntime(() => ({
-            disableDotnet6Compatibility: true,
-            configSrc: "./mono-config.json",
-        }));
+        const { MONO, BINDING, Module, RuntimeBuildInfo } = await createDotnetRuntime(() => {
+            console.log('user code in createDotnetRuntime')
+            return {
+                disableDotnet6Compatibility: true,
+                configSrc: "./mono-config.json",
+                preInit: () => { console.log('user code Module.preInit') },
+                preRun: () => { console.log('user code Module.preRun') },
+                onRuntimeInitialized: () => { console.log('user code Module.onRuntimeInitialized') },
+                postRun: () => { console.log('user code Module.postRun') },
+            }
+        });
+        console.log('after createDotnetRuntime')
 
         const testMeaning = BINDING.bind_static_method("[Wasm.Browser.CJS.Sample] Sample.Test:TestMeaning");
         const ret = testMeaning();

src/mono/wasm/runtime/dotnet.d.ts

@@ -155,6 +155,7 @@ declare type AssetEntry = {
     culture?: string;
     load_remote?: boolean;
     is_optional?: boolean;
+    buffer?: ArrayBuffer;
 };
 interface AssemblyEntry extends AssetEntry {
     name: "assembly";
@@ -196,7 +197,7 @@ declare type DotnetModuleConfig = {
     config?: MonoConfig | MonoConfigError;
     configSrc?: string;
     scriptDirectory?: string;
-    onConfigLoaded?: () => void;
+    onConfigLoaded?: (config: MonoConfig) => Promise<void>;
     onDotnetReady?: () => void;
     imports?: DotnetModuleConfigImports;
 } & Partial<EmscriptenModule>;

src/mono/wasm/runtime/exports.ts

@@ -31,9 +31,7 @@ import {
     mono_load_runtime_and_bcl_args, mono_wasm_load_config,
     mono_wasm_setenv, mono_wasm_set_runtime_options,
     mono_wasm_load_data_archive, mono_wasm_asm_loaded,
-    mono_wasm_pre_init,
-    mono_wasm_runtime_is_initialized,
-    mono_wasm_on_runtime_initialized
+    configure_emscripten_startup
 } from "./startup";
 import { mono_set_timeout, schedule_background_exec } from "./scheduling";
 import { mono_wasm_load_icu_data, mono_wasm_get_icudt_name } from "./icu";
@@ -66,7 +64,7 @@ import {
 import { create_weak_ref } from "./weak-ref";
 import { fetch_like, readAsync_like } from "./polyfills";
 import { EmscriptenModule } from "./types/emscripten";
-import { mono_on_abort, mono_run_main, mono_run_main_and_exit } from "./run";
+import { mono_run_main, mono_run_main_and_exit } from "./run";
 
 const MONO = {
     // current "public" MONO API
@@ -163,18 +161,6 @@ function initializeImportsAndExports(
         module.configSrc = "./mono-config.json";
     }
 
-    // these could be overriden on DotnetModuleConfig
-    if (!module.preInit) {
-        module.preInit = [];
-    } else if (typeof module.preInit === "function") {
-        module.preInit = [module.preInit];
-    }
-    if (!module.preRun) {
-        module.preRun = [];
-    } else if (typeof module.preRun === "function") {
-        module.preRun = [module.preRun];
-    }
-
     if (!module.print) {
         module.print = console.log.bind(console);
     }
@@ -254,37 +240,6 @@ function initializeImportsAndExports(
         warnWrap("removeRunDependency", () => module.removeRunDependency);
     }
 
-    // this is registration of the runtime pre_init, when user set configSrc
-    if (module.configSrc) {
-        module.preInit.push(async () => {
-            module.addRunDependency("mono_wasm_pre_init");
-            // execution order == [0] ==
-            await mono_wasm_pre_init();
-            module.removeRunDependency("mono_wasm_pre_init");
-        });
-    }
-
-    // if onRuntimeInitialized is set it's probably Blazor, we let them to do their own init sequence
-    if (!module.onRuntimeInitialized) {
-        // note this would keep running in async-parallel with emscripten's `run()` and `postRun()` 
-        // because it's loading files asynchronously and the emscripten is not awaiting onRuntimeInitialized
-        // execution order == [1] ==
-        module.onRuntimeInitialized = () => mono_wasm_on_runtime_initialized();
-
-        module.ready = module.ready.then(async () => {
-            // mono_wasm_runtime_is_initialized is set when finalize_startup is done
-            await mono_wasm_runtime_is_initialized;
-            // TODO we could take over Module.postRun and call it from here if necessary
-
-            // execution order == [2] ==
-            return exportedAPI;
-        });
-    }
-
-    if (!module.onAbort) {
-        module.onAbort = () => mono_on_abort;
-    }
-
     // this code makes it possible to find dotnet runtime on a page via global namespace, even when there are multiple runtimes at the same time
     let list: RuntimeList;
     if (!globalThisAny.getDotnetRuntime) {
@@ -296,6 +251,8 @@ function initializeImportsAndExports(
     }
     list.registerRuntime(exportedAPI);
 
+    configure_emscripten_startup(module, exportedAPI);
+
     return exportedAPI;
 }
 

src/mono/wasm/runtime/polyfills.ts

@@ -21,6 +21,8 @@ export async function fetch_like(url: string): Promise<Response> {
             };
         }
         else if (typeof (read) === "function") {
+            // note that it can't open files with unicode names, like Straße.xml
+            // https://bugs.chromium.org/p/v8/issues/detail?id=12541
             const arrayBuffer = new Uint8Array(read(url, "binary"));
             return <Response><any>{
                 ok: true,