[wasm-mt] Use asset loading for dotnet.worker.js; update WasmAppBuilder

src/mono/sample/wasm/browser-eventpipe/Makefile

@@ -2,6 +2,8 @@ TOP=../../../../..
 
 include ../wasm.mk
 
+override MSBUILD_ARGS+=/p:WasmEnablePerfTracing=true
+
 ifneq ($(AOT),)
 override MSBUILD_ARGS+=/p:RunAOTCompilation=true
 endif

src/mono/sample/wasm/browser-eventpipe/Wasm.Browser.EventPipe.Sample.csproj

@@ -13,6 +13,10 @@
     <NoWarn>CA2007</NoWarn> <!-- consider ConfigureAwait() -->
   </PropertyGroup>
 
+  <Target Name="CheckThreadsEnabled" BeforeTargets="Compile" >
+    <Warning Condition="'$(WasmEnableThreads)' != 'true' and '$(WasmEnablePerfTracing)' != 'true'" Text="This sample requires perftracing or threading" />
+  </Target>
+
   <PropertyGroup>
     <MonoDiagnosticsMock Condition="('$(MonoDiagnosticsMock)' == '') and ('$(Configuration)' == 'Debug')">true</MonoDiagnosticsMock>
   </PropertyGroup>

src/mono/sample/wasm/browser-threads/Makefile

@@ -2,6 +2,8 @@ TOP=../../../../..
 
 include ../wasm.mk
 
+override MSBUILD_ARGS+=/p:WasmEnableThreads=true
+
 ifneq ($(AOT),)
 override MSBUILD_ARGS+=/p:RunAOTCompilation=true
 endif

src/mono/sample/wasm/browser-threads/Wasm.Browser.Threads.Sample.csproj

@@ -16,6 +16,10 @@
   </PropertyGroup>
   <Target Name="RunSample" DependsOnTargets="RunSampleWithBrowser" />
 
+  <Target Name="CheckThreadsEnabled" BeforeTargets="Compile" >
+    <Warning Condition="'$(WasmEnableThreads)' != 'true'" Text="This sample requires threading" />
+  </Target>
+
   <!-- set the condition to false and you will get a CA1416 error about the call to Thread.Start from a browser-wasm project -->
   <ItemGroup Condition="true">
     <!-- TODO: some .props file that automates this.  Unfortunately just adding a ProjectReference to Microsoft.NET.WebAssembly.Threading.proj doesn't work - it ends up bundling the ref assemblies into the publish directory and breaking the app. -->

src/mono/wasm/build/WasmApp.targets

@@ -316,6 +316,12 @@
           DependsOnTargets="_WasmGenerateRuntimeConfig;_GetWasmGenerateAppBundleDependencies">
     <Error Condition="'$(WasmMainJSPath)' == ''" Text="%24(WasmMainJSPath) property needs to be set" />
 
+    <PropertyGroup>
+      <_WasmAppIncludeThreadsWorker Condition="'$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true'">true</_WasmAppIncludeThreadsWorker>
+      <!-- TODO: set this from some user-facing property?  -1 means use the default baked into dotnet.js -->
+      <_WasmPThreadPoolSize Condition="'$(_WasmPThreadPoolSize)' == '' and ('$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true')">-1</_WasmPThreadPoolSize>
+    </PropertyGroup>
+
     <RemoveDir Directories="$(WasmAppDir)" />
     <WasmAppBuilder
       AppDir="$(WasmAppDir)"
@@ -334,7 +340,10 @@
       ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)"
       ExtraConfig="@(WasmExtraConfig)"
       NativeAssets="@(WasmNativeAsset)"
-      DebugLevel="$(WasmDebugLevel)">
+      DebugLevel="$(WasmDebugLevel)"
+      IncludeThreadsWorker="$(_WasmAppIncludeThreadsWorker)"
+      PThreadPoolSize="$(_WasmPThreadPoolSize)"
+      >
       <Output TaskParameter="FileWrites" ItemName="FileWrites" />
     </WasmAppBuilder>
 

src/mono/wasm/runtime/dotnet.d.ts

@@ -81,6 +81,7 @@ declare type MonoConfig = {
     coverageProfilerOptions?: CoverageProfilerOptions;
     ignorePdbLoadErrors?: boolean;
     waitForDebugger?: number;
+    pthreadPoolSize?: number;
 };
 interface ResourceRequest {
     name: string;

src/mono/wasm/runtime/es6/dotnet.es6.lib.js

@@ -26,6 +26,7 @@ let __dotnet_replacement_PThread = ${usePThreads} ? {} : undefined;
 if (${usePThreads}) {
     __dotnet_replacement_PThread.loadWasmModuleToWorker = PThread.loadWasmModuleToWorker;
     __dotnet_replacement_PThread.threadInitTLS = PThread.threadInitTLS;
+    __dotnet_replacement_PThread.allocateUnusedWorker = PThread.allocateUnusedWorker;
 }
 let __dotnet_replacements = {scriptUrl: import.meta.url, fetch: globalThis.fetch, require, updateGlobalBufferAndViews, pthreadReplacements: __dotnet_replacement_PThread};
 if (ENVIRONMENT_IS_NODE) {
@@ -47,6 +48,7 @@ var noExitRuntime = __dotnet_replacements.noExitRuntime;
 if (${usePThreads}) {
     PThread.loadWasmModuleToWorker = __dotnet_replacements.pthreadReplacements.loadWasmModuleToWorker;
     PThread.threadInitTLS = __dotnet_replacements.pthreadReplacements.threadInitTLS;
+    PThread.allocateUnusedWorker = __dotnet_replacements.pthreadReplacements.allocateUnusedWorker;
 }
 `,
 };

src/mono/wasm/runtime/polyfills.ts

@@ -2,8 +2,7 @@ import BuildConfiguration from "consts:configuration";
 import MonoWasmThreads from "consts:monoWasmThreads";
 import { ENVIRONMENT_IS_NODE, ENVIRONMENT_IS_SHELL, ENVIRONMENT_IS_WEB, ENVIRONMENT_IS_WORKER, INTERNAL, Module, runtimeHelpers } from "./imports";
 import { afterUpdateGlobalBufferAndViews } from "./memory";
-import { afterLoadWasmModuleToWorker } from "./pthreads/browser";
-import { afterThreadInitTLS } from "./pthreads/worker";
+import { replaceEmscriptenPThreadLibrary } from "./pthreads/shared/emscripten-replacements";
 import { DotnetModuleConfigImports, EarlyReplacements } from "./types";
 
 let node_fs: any | undefined = undefined;
@@ -173,16 +172,7 @@ export function init_polyfills(replacements: EarlyReplacements): void {
     // threads
     if (MonoWasmThreads) {
         if (replacements.pthreadReplacements) {
-            const originalLoadWasmModuleToWorker = replacements.pthreadReplacements.loadWasmModuleToWorker;
-            replacements.pthreadReplacements.loadWasmModuleToWorker = (worker: Worker, onFinishedLoading: Function): void => {
-                originalLoadWasmModuleToWorker(worker, onFinishedLoading);
-                afterLoadWasmModuleToWorker(worker);
-            };
-            const originalThreadInitTLS = replacements.pthreadReplacements.threadInitTLS;
-            replacements.pthreadReplacements.threadInitTLS = (): void => {
-                originalThreadInitTLS();
-                afterThreadInitTLS();
-            };
+            replaceEmscriptenPThreadLibrary(replacements.pthreadReplacements);
         }
     }
 
@@ -297,4 +287,4 @@ function isPathAbsolute(path: string): boolean {
     // windows file:///C:/x.json
     // windows http://C:/x.json
     return protocolRx.test(path);
-}
+}

src/mono/wasm/runtime/pthreads/browser/index.ts

@@ -1,10 +1,12 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-import { Module } from "../../imports";
-import { pthread_ptr, MonoWorkerMessageChannelCreated, isMonoWorkerMessageChannelCreated, monoSymbol } from "../shared";
+import { MonoWorkerMessageChannelCreated, isMonoWorkerMessageChannelCreated, monoSymbol } from "../shared";
+import { pthread_ptr } from "../shared/types";
 import { MonoThreadMessage } from "../shared";
 import { PromiseController, createPromiseController } from "../../promise-controller";
+import { MonoConfig, mono_assert } from "../../types";
+import Internals from "../shared/emscripten-internals";
 
 const threads: Map<pthread_ptr, Thread> = new Map();
 
@@ -103,21 +105,40 @@ export function afterLoadWasmModuleToWorker(worker: Worker): void {
     console.debug("MONO_WASM: afterLoadWasmModuleToWorker added message event handler", worker);
 }
 
-/// These utility functions dig into Emscripten internals
-const Internals = {
-    getWorker: (pthread_ptr: pthread_ptr): Worker => {
-        // see https://github.com/emscripten-core/emscripten/pull/16239
-        return (<any>Module).PThread.pthreads[pthread_ptr].worker;
-    },
-    getThreadId: (worker: Worker): pthread_ptr | undefined => {
-        /// See library_pthread.js in Emscripten.
-        /// They hang a "pthread" object from the worker if the worker is running a thread, and remove it when the thread stops by doing `pthread_exit` or when it's joined using `pthread_join`.
-        const emscriptenThreadInfo = (<any>worker)["pthread"];
-        if (emscriptenThreadInfo === undefined) {
-            return undefined;
-        }
-        return emscriptenThreadInfo.threadInfoStruct;
+/// We call on the main thread this during startup to pre-allocate a pool of pthread workers.
+/// At this point asset resolution needs to be working (ie we loaded MonoConfig).
+/// This is used instead of the Emscripten PThread.initMainThread because we call it later.
+export function preAllocatePThreadWorkerPool(defaultPthreadPoolSize: number, config: MonoConfig): void {
+    const poolSizeSpec = config?.pthreadPoolSize;
+    let n: number;
+    if (poolSizeSpec === undefined) {
+        n = defaultPthreadPoolSize;
+    } else {
+        mono_assert(typeof poolSizeSpec === "number", "pthreadPoolSize must be a number");
+        if (poolSizeSpec < 0)
+            n = defaultPthreadPoolSize;
+        else
+            n = poolSizeSpec;
     }
-};
-
+    for (let i = 0; i < n; i++) {
+        Internals.allocateUnusedWorker();
+    }
+}
 
+/// We call this on the main thread during startup once we fetched WasmModule.
+/// This sends a message to each pre-allocated worker to load the WasmModule and dotnet.js and to set up
+/// message handling.
+/// This is used instead of the Emscripten "receiveInstance" in "createWasm" because that code is
+/// conditioned on a non-zero PTHREAD_POOL_SIZE (but we set it to 0 to avoid early worker allocation).
+export async function instantiateWasmPThreadWorkerPool(): Promise<void> {
+    // this is largely copied from emscripten's "receiveInstance" in "createWasm" in "src/preamble.js"
+    const workers = Internals.getUnusedWorkerPool();
+    const allLoaded = createPromiseController<void>();
+    let leftToLoad = workers.length;
+    workers.forEach((w) => {
+        Internals.loadWasmModuleToWorker(w, function () {
+            if (!--leftToLoad) allLoaded.promise_control.resolve();
+        });
+    });
+    await allLoaded.promise;
+}

src/mono/wasm/runtime/pthreads/shared/emscripten-internals.ts

@@ -0,0 +1,76 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+import { Module } from "../../imports";
+import { pthread_ptr } from "./types";
+
+/** @module emscripten-internals accessors to the functions in the emscripten PThreads library, including
+ * the low-level representations of {@linkcode pthread_ptr} thread info structs, etc.
+ * Additionally, note that some of these functions are replaced by {@linkcode file://./emscripten-replacements.ts}.
+ * These have a hard dependency on the version of Emscripten that we are using and may need to be kept in sync with
+ *    {@linkcode file://./../../../emsdk/upstream/emscripten/src/library_pthread.js}
+ */
+
+// This is what we know about the Emscripten PThread library
+interface PThreadLibrary {
+    unusedWorkers: Worker[];
+    pthreads: PThreadInfoMap;
+    allocateUnusedWorker: () => void;
+    loadWasmModuleToWorker: (worker: Worker, onFinishedLoading?: (worker: Worker) => void) => void;
+}
+
+interface EmscriptenPThreadInfo {
+    threadInfoStruct: pthread_ptr;
+}
+
+/// N.B. emscripten deletes the `pthread` property from the worker when it is not actively running a pthread
+interface PThreadWorker extends Worker {
+    pthread: EmscriptenPThreadInfo;
+}
+
+interface PThreadObject {
+    worker: PThreadWorker;
+}
+
+interface PThreadInfoMap {
+    [key: pthread_ptr]: PThreadObject | undefined;
+}
+
+
+function isRunningPThreadWorker(w: Worker): w is PThreadWorker {
+    return (<any>w).pthread !== undefined;
+}
+
+/// These utility functions dig into Emscripten internals
+const Internals = {
+    get modulePThread(): PThreadLibrary {
+        return (<any>Module).PThread as PThreadLibrary;
+    },
+    getWorker: (pthread_ptr: pthread_ptr): PThreadWorker | undefined => {
+        // see https://github.com/emscripten-core/emscripten/pull/16239
+        return Internals.modulePThread.pthreads[pthread_ptr]?.worker;
+    },
+    getThreadId: (worker: Worker): pthread_ptr | undefined => {
+        /// See library_pthread.js in Emscripten.
+        /// They hang a "pthread" object from the worker if the worker is running a thread, and remove it when the thread stops by doing `pthread_exit` or when it's joined using `pthread_join`.
+        if (!isRunningPThreadWorker(worker))
+            return undefined;
+        const emscriptenThreadInfo = worker.pthread;
+        return emscriptenThreadInfo.threadInfoStruct;
+    },
+    allocateUnusedWorker: (): void => {
+        /// See library_pthread.js in Emscripten.
+        /// This function allocates a new worker and adds it to the pool of workers.
+        /// It's called when the pool of workers is empty and a new thread is created.
+        Internals.modulePThread.allocateUnusedWorker();
+    },
+    getUnusedWorkerPool: (): Worker[] => {
+        return Internals.modulePThread.unusedWorkers;
+    },
+    loadWasmModuleToWorker: (worker: Worker, onFinishedLoading: () => void): void => {
+        Internals.modulePThread.loadWasmModuleToWorker(worker, onFinishedLoading);
+    }
+};
+
+
+export default Internals;

src/mono/wasm/runtime/pthreads/shared/emscripten-replacements.ts

@@ -0,0 +1,43 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+import MonoWasmThreads from "consts:monoWasmThreads";
+import { PThreadReplacements } from "../../types";
+import { afterLoadWasmModuleToWorker } from "../browser";
+import { afterThreadInitTLS } from "../worker";
+import Internals from "./emscripten-internals";
+import { resolve_asset_path } from "../../assets";
+import { mono_assert } from "../../types";
+
+
+/** @module emscripten-replacements Replacements for individual functions in the emscripten PThreads library.
+ * These have a hard dependency on the version of Emscripten that we are using and may need to be kept in sync with
+ *    {@linkcode file://./../../../emsdk/upstream/emscripten/src/library_pthread.js}
+ */
+
+export function replaceEmscriptenPThreadLibrary(replacements: PThreadReplacements): void {
+    if (MonoWasmThreads) {
+        const originalLoadWasmModuleToWorker = replacements.loadWasmModuleToWorker;
+        replacements.loadWasmModuleToWorker = (worker: Worker, onFinishedLoading?: (worker: Worker) => void): void => {
+            originalLoadWasmModuleToWorker(worker, onFinishedLoading);
+            afterLoadWasmModuleToWorker(worker);
+        };
+        const originalThreadInitTLS = replacements.threadInitTLS;
+        replacements.threadInitTLS = (): void => {
+            originalThreadInitTLS();
+            afterThreadInitTLS();
+        };
+        // const originalAllocateUnusedWorker = replacements.allocateUnusedWorker;
+        replacements.allocateUnusedWorker = replacementAllocateUnusedWorker;
+    }
+}
+
+/// We replace Module["PThreads"].allocateUnusedWorker with this version that knows about assets
+function replacementAllocateUnusedWorker(): void {
+    console.debug("MONO_WASM: replacementAllocateUnusedWorker");
+    const asset = resolve_asset_path("js-module-threads");
+    const uri = asset.resolvedUrl;
+    mono_assert(uri !== undefined, "could not resolve the uri for the js-module-threads asset");
+    const worker = new Worker(uri);
+    Internals.getUnusedWorkerPool().push(worker);
+}

src/mono/wasm/runtime/pthreads/shared/index.ts

@@ -2,9 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import { Module } from "../../imports";
-
-/// pthread_t in C
-export type pthread_ptr = number;
+import { pthread_ptr } from "./types";
 
 export interface PThreadInfo {
     readonly pthread_id: pthread_ptr;

src/mono/wasm/runtime/pthreads/shared/tsconfig.json

@@ -1,3 +1,8 @@
 {
-    "extends": "../../tsconfig.shared.json"
+    "extends": "../../tsconfig.shared.json",
+    "include": [
+        "../../**/*.ts",
+        "../../**/*.d.ts"
+    ]
+
 }

src/mono/wasm/runtime/pthreads/shared/types.ts

@@ -0,0 +1,5 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+/// pthread_t in C
+export type pthread_ptr = number;

src/mono/wasm/runtime/pthreads/worker/events.ts

@@ -1,5 +1,6 @@
 import MonoWasmThreads from "consts:monoWasmThreads";
-import type { pthread_ptr, PThreadInfo, MonoThreadMessage } from "../shared";
+import type { pthread_ptr } from "../shared/types";
+import type { PThreadInfo, MonoThreadMessage } from "../shared";
 
 /// Identification of the current thread executing on a worker
 export interface PThreadSelf extends PThreadInfo {

src/mono/wasm/runtime/pthreads/worker/index.ts

@@ -5,7 +5,8 @@
 
 import MonoWasmThreads from "consts:monoWasmThreads";
 import { Module, ENVIRONMENT_IS_PTHREAD } from "../../imports";
-import { makeChannelCreatedMonoMessage, pthread_ptr } from "../shared";
+import { makeChannelCreatedMonoMessage } from "../shared";
+import type { pthread_ptr } from "../shared/types";
 import { mono_assert, is_nullish } from "../../types";
 import type { MonoThreadMessage } from "../shared";
 import {