Add example with WASM audio worklet

lukaslihotzki · lukaslihotzki · commit 33a8682d7ab5 · 2022-08-03T03:15:41.000+02:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -77,6 +77,7 @@ members = [
   "examples/raytrace-parallel",
   "examples/request-animation-frame",
   "examples/todomvc",
+  "examples/wasm-audio-worklet",
   "examples/wasm-in-wasm",
   "examples/wasm-in-wasm-imports",
   "examples/wasm-in-web-worker",
diff --git a/examples/wasm-audio-worklet/Cargo.toml b/examples/wasm-audio-worklet/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "wasm-audio-worklet"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+console_log = "0.2.0"
+js-sys = "0.3.59"
+wasm-bindgen = "0.2.82"
+wasm-bindgen-futures = "0.4.32"
+web-sys = {version = "0.3.59", features = ["AudioContext", "AudioDestinationNode", "AudioWorklet", "AudioWorkletNode", "AudioWorkletNodeOptions", "Blob", "BlobPropertyBag", "Document", "HtmlInputElement", "HtmlLabelElement", "Url", "Window"]}
+
+[lib]
+crate-type = ["cdylib", "rlib"]
diff --git a/examples/wasm-audio-worklet/README.md b/examples/wasm-audio-worklet/README.md
@@ -0,0 +1,28 @@
+# wasm-audio-worklet
+
+[View documentation for this example online][dox] or [View compiled example
+online][compiled]
+
+[dox]: https://rustwasm.github.io/docs/wasm-bindgen/examples/wasm-audio-worklet.html
+[compiled]: https://rustwasm.github.io/wasm-bindgen/exbuild/wasm-audio-worklet/
+
+You can build the example locally with:
+
+```
+$ ./build.sh
+```
+
+(or running the commands on Windows manually)
+
+and then visiting http://localhost:8080 in a browser should run the example!
+
+## dependent_module
+
+This example shows how libraries can create worklets with minimal extra effort
+for application developers. No extra scripts or specific JS class names
+are needed. Just the build commands need to be changed to enable threads,
+and the server needs to send extra headers to enable site isolation.
+
+dependent_module relies on ES6 `import`, which is
+[currently unavailable](https://bugzilla.mozilla.org/show_bug.cgi?id=1572644)
+for worklets in Firefox. The example works in Chrome.
diff --git a/examples/wasm-audio-worklet/build.sh b/examples/wasm-audio-worklet/build.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -ex
+
+# A couple of steps are necessary to get this build working which makes it slightly
+# nonstandard compared to most other builds.
+#
+# * First, the Rust standard library needs to be recompiled with atomics
+#   enabled. to do that we use Cargo's unstable `-Zbuild-std` feature.
+#
+# * Next we need to compile everything with the `atomics` and `bulk-memory`
+#   features enabled, ensuring that LLVM will generate atomic instructions,
+#   shared memory, passive segments, etc.
+
+RUSTFLAGS='-C target-feature=+atomics,+bulk-memory,+mutable-globals' \
+  cargo build --target wasm32-unknown-unknown --release -Z build-std=std,panic_abort
+
+cargo run -p wasm-bindgen-cli -- \
+  ../../target/wasm32-unknown-unknown/release/wasm_audio_worklet.wasm \
+  --out-dir . \
+  --target web
+
+python3 server.py
diff --git a/examples/wasm-audio-worklet/index.html b/examples/wasm-audio-worklet/index.html
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>WASM audio worklet</title>
+    </head>
+    <body>
+        <script type="module">
+            import init, {web_main} from "./wasm_audio_worklet.js";
+            async function run() {
+                await init();
+                web_main();
+            }
+            run();
+        </script>
+    </body>
+</html>
diff --git a/examples/wasm-audio-worklet/server.py b/examples/wasm-audio-worklet/server.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+from http.server import HTTPServer, SimpleHTTPRequestHandler, test
+import sys
+
+class RequestHandler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_header('Cross-Origin-Opener-Policy', 'same-origin')
+        self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
+        SimpleHTTPRequestHandler.end_headers(self)
+
+if __name__ == '__main__':
+    test(RequestHandler, HTTPServer, port=int(sys.argv[1]) if len(sys.argv) > 1 else 8000)
diff --git a/examples/wasm-audio-worklet/src/dependent_module.rs b/examples/wasm-audio-worklet/src/dependent_module.rs
@@ -0,0 +1,43 @@
+use js_sys::{Array, JsString};
+use wasm_bindgen::prelude::*;
+use web_sys::{Blob, BlobPropertyBag, Url};
+
+// This is a not-so-clean approach to get the current bindgen JS module URL
+// in Rust.
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen]
+    type ImportMeta;
+
+    #[wasm_bindgen(method, getter)]
+    fn url(this: &ImportMeta) -> JsString;
+
+    #[wasm_bindgen(js_namespace = import, js_name = meta)]
+    static IMPORT_META: ImportMeta;
+}
+
+pub fn on_the_fly(code: &str) -> Result<String, JsValue> {
+    let header = format!(
+        "import init, * as bindgen from '{}';\n\n",
+        IMPORT_META.url(),
+    );
+    Url::create_object_url_with_blob(&Blob::new_with_str_sequence_and_options(
+        &Array::of2(&JsValue::from(header.as_str()), &JsValue::from(code)),
+        &BlobPropertyBag::new().type_("text/javascript"),
+    )?)
+}
+
+// dependent_module! takes a local file name to a JS module as input and
+// returns a URL to a slightly modified module in run time. This modified module
+// has an additional import statement in the header that imports the current
+// bindgen JS module under the `bindgen` alias.
+// How this URL is produced does not matter for the macro user. on_the_fly
+// creates a blob URL in run time. A better, more sophisticated solution
+// would add wasm_bindgen support to put such a module in pkg/ during build time
+// and return a relative URL to this file instead.
+#[macro_export]
+macro_rules! dependent_module {
+    ($file_name:expr) => {
+        crate::dependent_module::on_the_fly(include_str!($file_name))
+    };
+}
diff --git a/examples/wasm-audio-worklet/src/get_bindgen_transferable.js b/examples/wasm-audio-worklet/src/get_bindgen_transferable.js
@@ -0,0 +1,6 @@
+export function get_bindgen_transferable(o) {
+    return {
+        type: Object.getPrototypeOf(o).constructor.name,
+        ptr: o.ptr,
+    };
+}
diff --git a/examples/wasm-audio-worklet/src/gui.rs b/examples/wasm-audio-worklet/src/gui.rs
@@ -0,0 +1,48 @@
+use crate::oscillator::{Oscillator, Params};
+use crate::wasm_audio::wasm_audio;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::{closure::Closure, JsCast, JsValue};
+use web_sys::{HtmlInputElement, HtmlLabelElement};
+
+#[wasm_bindgen]
+pub async fn web_main() {
+    // On the application level, most audio worklet internals are abstracted
+    // by wasm_audio:
+    let params: &'static Params = Box::leak(Box::new(Params::default()));
+    let osc = Oscillator::new(&params);
+    let ctx = wasm_audio(osc.into()).await.unwrap();
+
+    let window = web_sys::window().unwrap();
+    let document = window.document().unwrap();
+    let body = document.body().unwrap();
+
+    let volume = add_slider(&document, &body, "Volume:").unwrap();
+    let frequency = add_slider(&document, &body, "Frequency:").unwrap();
+    volume.set_value("0");
+    frequency.set_min("20");
+    frequency.set_value("60");
+
+    let listener = Closure::<dyn FnMut(_)>::new(move |_: web_sys::Event| {
+        params.set_frequency(frequency.value().parse().unwrap());
+        params.set_volume(volume.value().parse().unwrap());
+        ctx.resume().unwrap();
+    })
+    .into_js_value();
+
+    body.add_event_listener_with_callback("input", listener.as_ref().unchecked_ref())
+        .unwrap();
+}
+
+fn add_slider(
+    document: &web_sys::Document,
+    body: &web_sys::HtmlElement,
+    name: &str,
+) -> Result<HtmlInputElement, JsValue> {
+    let input: HtmlInputElement = document.create_element("input")?.unchecked_into();
+    let label: HtmlLabelElement = document.create_element("label")?.unchecked_into();
+    input.set_type("range");
+    label.set_text_content(Some(name));
+    label.append_child(&input)?;
+    body.append_child(&label)?;
+    Ok(input)
+}
diff --git a/examples/wasm-audio-worklet/src/lib.rs b/examples/wasm-audio-worklet/src/lib.rs
@@ -0,0 +1,4 @@
+mod dependent_module;
+mod gui;
+mod oscillator;
+mod wasm_audio;
diff --git a/examples/wasm-audio-worklet/src/oscillator.rs b/examples/wasm-audio-worklet/src/oscillator.rs
@@ -0,0 +1,58 @@
+// WASM audio processors can be implemented in Rust without knowing
+// about audio worklets. Audio processors just need to expose the process method
+// through wasm_bindgen. Currently, this interface cannot be assured by traits.
+
+use std::sync::atomic::{AtomicU8, Ordering};
+use wasm_bindgen::prelude::*;
+
+// Let's implement a simple sine oscillator with variable frequency and volume.
+#[wasm_bindgen]
+pub struct Oscillator {
+    params: &'static Params,
+    accumulator: u32,
+}
+
+impl Oscillator {
+    pub fn new(params: &'static Params) -> Self {
+        Self {
+            params,
+            accumulator: 0,
+        }
+    }
+}
+
+#[wasm_bindgen]
+impl Oscillator {
+    pub fn process(&mut self, output: &mut [f32]) -> bool {
+        // This method is called in the audio process thread.
+        // All imports are set, so host functionality available in worklets
+        // (for example, logging) can be used:
+        // `web_sys::console::log_1(&JsValue::from(output.len()));`
+        // Note that currently TextEncoder and TextDecoder are stubs, so passing
+        // strings may not work in this thread.
+        for a in output {
+            let frequency = self.params.frequency.load(Ordering::Relaxed);
+            let volume = self.params.volume.load(Ordering::Relaxed);
+            self.accumulator += u32::from(frequency);
+            *a = (self.accumulator as f32 / 512.).sin() * (volume as f32 / 100.);
+        }
+        true
+    }
+}
+
+#[derive(Default)]
+pub struct Params {
+    // Use atomics for parameters so they can be set in the main thread and
+    // fetched by the audio process thread without further synchronization.
+    frequency: AtomicU8,
+    volume: AtomicU8,
+}
+
+impl Params {
+    pub fn set_frequency(&self, frequency: u8) {
+        self.frequency.store(frequency, Ordering::Relaxed);
+    }
+    pub fn set_volume(&self, volume: u8) {
+        self.volume.store(volume, Ordering::Relaxed);
+    }
+}
diff --git a/examples/wasm-audio-worklet/src/polyfill.js b/examples/wasm-audio-worklet/src/polyfill.js
@@ -0,0 +1,18 @@
+if (!globalThis.TextDecoder) {
+    globalThis.TextDecoder = class TextDecoder {
+        decode() {
+            return "";
+        }
+    };
+}
+
+if (!globalThis.TextEncoder) {
+    globalThis.TextEncoder = class TextEncoder {
+        encode() {
+            return new Uint8Array(0);
+        }
+    };
+}
+
+export function nop() {
+}
diff --git a/examples/wasm-audio-worklet/src/wasm_audio.rs b/examples/wasm-audio-worklet/src/wasm_audio.rs
@@ -0,0 +1,57 @@
+use crate::dependent_module;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsValue;
+use wasm_bindgen_futures::JsFuture;
+use web_sys::{AudioContext, AudioWorkletNode, AudioWorkletNodeOptions};
+
+// Use wasm_audio if you have a single wasm audio processor in your application
+// whose samples should be played directly.
+pub async fn wasm_audio(wasm_processor: JsValue) -> Result<AudioContext, JsValue> {
+    let ctx = AudioContext::new()?;
+    prepare_wasm_audio(&ctx).await?;
+    let node = wasm_audio_node(&ctx, wasm_processor)?;
+    node.connect_with_audio_node(&ctx.destination())?;
+    Ok(ctx)
+}
+
+// wasm_audio_node creates an AudioWorkletNode running a wasm audio processor.
+// Remember to call prepare_wasm_audio once on your context before calling
+// this function.
+pub fn wasm_audio_node(
+    ctx: &AudioContext,
+    wasm_processor: JsValue,
+) -> Result<AudioWorkletNode, JsValue> {
+    AudioWorkletNode::new_with_options(
+        &ctx,
+        "WasmProcessor",
+        &AudioWorkletNodeOptions::new().processor_options(Some(&js_sys::Array::of3(
+            &wasm_bindgen::module(),
+            &wasm_bindgen::memory(),
+            &get_bindgen_transferable(wasm_processor),
+        ))),
+    )
+}
+
+pub async fn prepare_wasm_audio(ctx: &AudioContext) -> Result<(), JsValue> {
+    nop();
+    let mod_url = dependent_module!("worklet_async.js")?;
+    JsFuture::from(ctx.audio_worklet()?.add_module(&mod_url)?).await?;
+    Ok(())
+}
+
+// TextEncoder and TextDecoder are not available in Audio Worklets, but there
+// is a dirty workaround: Import polyfill.js to install stub implementations
+// of these classes in globalThis.
+#[wasm_bindgen(module = "/src/polyfill.js")]
+extern "C" {
+    fn nop();
+}
+
+#[wasm_bindgen(module = "/src/get_bindgen_transferable.js")]
+extern "C" {
+    // When creating a JsValue from a wasm_bindgen struct, the resulting object
+    // contains just the numeric pointer after structured clone. To call methods
+    // on objects, the type must be known too. get_bindgen_transferable creates
+    // a JS object that contains both the type and the numeric pointer.
+    fn get_bindgen_transferable(obj: JsValue) -> JsValue;
+}
diff --git a/examples/wasm-audio-worklet/src/worklet.js b/examples/wasm-audio-worklet/src/worklet.js
@@ -0,0 +1,14 @@
+// Doesn't work until https://github.com/rustwasm/wasm-bindgen/pull/3016 is available.
+// Meanwhile, worklet_async can be used instead.
+
+registerProcessor("WasmProcessor", class WasmProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super();
+        let [module, memory, {type, ptr}] = options.processorOptions;
+        bindgen.initSync(module, memory);
+        this.processor = bindgen[type].__wrap(ptr);
+    }
+    process(inputs, outputs) {
+        return this.processor.process(outputs[0][0]);
+    }
+});
diff --git a/examples/wasm-audio-worklet/src/worklet_async.js b/examples/wasm-audio-worklet/src/worklet_async.js
@@ -0,0 +1,13 @@
+registerProcessor("WasmProcessor", class WasmProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super();
+        let [module, memory, {type, ptr}] = options.processorOptions;
+        init(module, memory).then(() => {
+            this.processor = bindgen[type].__wrap(ptr);
+        });
+    }
+    process(inputs, outputs) {
+        if (!this.processor) return true;
+        return this.processor.process(outputs[0][0]);
+    }
+});
