From e11288c580901a9eb9798755ef712a6a6e92f291 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Sat, 7 Sep 2019 18:47:43 -0700 Subject: [PATCH 01/20] wip --- emcc.py | 4 ++ src/library_wasi.js | 14 +++++ src/modules.js | 4 ++ src/preamble.js | 4 ++ src/settings.js | 5 ++ system/lib/wasi.c | 124 +++++++++++++++++++++++++++++++++++++++++++ tools/shared.py | 11 ++-- tools/system_libs.py | 11 ++++ 8 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/library_wasi.js create mode 100644 system/lib/wasi.c diff --git a/emcc.py b/emcc.py index d82537d2438e0..bd81adfd31f8e 100755 --- a/emcc.py +++ b/emcc.py @@ -1728,6 +1728,10 @@ def check_human_readable_list(items): if shared.Settings.MINIMAL_RUNTIME and not shared.Settings.WASM: options.separate_asm = True + if shared.Settings.WASI: + # we cannot legalize the JS FFI when using wasi, as it can run standalone in wasm + shared.Settings.LEGALIZE_JS_FFI = 0 + if shared.Settings.WASM_BACKEND: if shared.Settings.SIMD: newargs.append('-msimd128') diff --git a/src/library_wasi.js b/src/library_wasi.js new file mode 100644 index 0000000000000..8f082e0fbd9a0 --- /dev/null +++ b/src/library_wasi.js @@ -0,0 +1,14 @@ +/* + * Copyright 2019 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +mergeInto(LibraryManager.library, { + proc_exit__deps: ['exit'], + proc_exit: function(code) { + return _exit(code); + }, +}); + diff --git a/src/modules.js b/src/modules.js index 45c06975796d4..13683eb67d7bd 100644 --- a/src/modules.js +++ b/src/modules.js @@ -154,6 +154,10 @@ var LibraryManager = { libraries.push('library_glemu.js'); } + if (WASI) { + libraries.push('library_wasi.js'); + } + libraries = libraries.concat(additionalLibraries); if (BOOTSTRAPPING_STRUCT_INFO) libraries = ['library_bootstrap_structInfo.js', 'library_formatString.js']; diff --git a/src/preamble.js b/src/preamble.js index 599b25f454658..0257a6b60bb18 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1028,6 +1028,10 @@ function createWasm(env) { exports = Asyncify.instrumentWasmExports(exports); #endif Module['asm'] = exports; +#if WASI + // In wasi the memory is exported. + updateGlobalBufferAndViews(exports['memory'].buffer); +#endif #if USE_PTHREADS // Keep a reference to the compiled module so we can post it to the workers. wasmModule = module; diff --git a/src/settings.js b/src/settings.js index 9b65ee710cff5..8398749d649f4 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1072,6 +1072,11 @@ var USE_GLFW = 2; // port, which can useful for local dev work on binaryen itself). var WASM = 1; +// Whether to use the WASI APIs. This still emits a JS+wasm combination, but +// the wasm part can run as a wasi executable inside a wasi runtime, without +// any JS - basically, the JS is just a convenient way to run it on the Web. +var WASI = 0; + // Whether to use the WebAssembly backend that is in development in LLVM. You // should not set this yourself, instead set EMCC_WASM_BACKEND=1 in the // environment. diff --git a/system/lib/wasi.c b/system/lib/wasi.c new file mode 100644 index 0000000000000..b18d66e2a3445 --- /dev/null +++ b/system/lib/wasi.c @@ -0,0 +1,124 @@ +/* + * Copyright 2019 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + * + * C++ exception handling support stubs. This is included when exception + * throwing is disabled - so no exceptions should exist at all. If the code still + * uses them, these stubs will throw at runtime. + */ + +#include +#include +#include +#include + +#include "wasi.h" + +// WASI APIs + +// libc + +void exit(int status) { + __wasi_proc_exit(status); + __builtin_unreachable(); +} + +void abort() { + exit(1); +} + +// Musl lock internals. As we assume wasi is single-threaded for now, these +// are no-ops. + +void __lock(void* ptr) {} +void __unlock(void* ptr) {} + +// Emscripten additions + +void *emscripten_memcpy_big(void *restrict dest, const void *restrict src, size_t n) { + // This normally calls out into JS which can do a single fast operation, + // but with wasi we can't do that. As this is called when n >= 8192, we + // can just split into smaller calls. + // TODO optimize, maybe build our memcpy with a wasi variant? + const int CHUNK = 8192; + unsigned char* d = (unsigned char*)dest; + unsigned char* s = (unsigned char*)src; + while (n > 0) { + size_t curr_n = n; + if (curr_n > CHUNK) curr_n = CHUNK; + memcpy(d, s, curr_n); + d += CHUNK; + s += CHUNK; + n -= curr_n; + } + return dest; +} + +// I/O syscalls - we support printf etc., but no filesystem access for now. + +static intptr_t _vararg; + +static void set_vararg(intptr_t vararg) { + _vararg = vararg; +} + +static int get_vararg_i32() { + int ret = *(int*)_vararg; + _vararg += 4; + return ret; +} + +int __syscall6(int id, intptr_t vararg) { // close + return 0; +} + +int __syscall54(int id, intptr_t vararg) { // ioctl + return 0; +} + +int __syscall140(int id, intptr_t vararg) { // llseek + return 0; +} + +int __syscall146(int id, intptr_t vararg) { // writev + set_vararg(vararg); + // hack to support printf, similar to library_syscalls.js handling of SYSCALLS_REQUIRE_FILESYSTEM=0 + int stream = get_vararg_i32(); + // Luckily iovs are identical in musl and wasi + struct __wasi_ciovec_t* iov = (struct __wasi_ciovec_t*)get_vararg_i32(); + size_t iovcnt = get_vararg_i32(); + size_t ret = 0; + for (int i = 0; i < iovcnt; i++) { + size_t num; + int result = __wasi_fd_write(stream, iov, 1, &num); + // TODO: error handling + ret += num; + iov++; + } + return ret; +} + +// Memory management + +extern intptr_t emscripten_get_sbrk_ptr(); + +static const size_t WASM_PAGE_SIZE = 65536; + +void *sbrk(intptr_t increment) { + intptr_t old_brk = emscripten_get_sbrk_ptr(); + // TODO: overflow checks + intptr_t updated_brk = old_brk + increment; + uintptr_t old_size = __builtin_wasm_memory_size(0) * WASM_PAGE_SIZE; + if (updated_brk >= old_size) { + // Try to grow memory. + intptr_t diff = updated_brk - old_size; + uintptr_t result = __builtin_wasm_memory_grow(0, (diff + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE); + if (result == SIZE_MAX) { + errno = ENOMEM; + return (void*)-1; + } + } + return (void*)old_brk; +} diff --git a/tools/shared.py b/tools/shared.py index 1dcaa7a143de8..dce4eda88089e 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1793,11 +1793,15 @@ def link_lld(args, target, opts=[], lto_level=0): '-o', target, '--allow-undefined', - '--import-memory', - '--import-table', '--lto-O%d' % lto_level, ] + args + # wasi does not import the memory, but for JS it is convenient to do so, + # as it allows us to set up memory even before the wasm module arrives + if not Settings.WASI: + cmd.append('--import-memory') + cmd.append('--import-table') + if Settings.USE_PTHREADS: cmd.append('--shared-memory') @@ -2465,7 +2469,8 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if Settings.EMITTING_JS and not Settings.AUTODEBUG: + # For wasi, in theory we could minify the non-wasi ones, but the point of wasi is to not have such things anyhow. + if Settings.EMITTING_JS and not Settings.AUTODEBUG and not Settings.WASI: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file diff --git a/tools/system_libs.py b/tools/system_libs.py index 678a2f7245c13..112b91cdb9308 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1157,6 +1157,14 @@ class libasan_rt_wasm(SanitizerLibrary): src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'asan'] +class libwasi(WasiLibrary): + name = 'libwasi' + + cflags = ['-Os'] + src_dir = ['system', 'lib'] + src_files = ['wasi.c'] + + # If main() is not in EXPORTED_FUNCTIONS, it may be dce'd out. This can be # confusing, so issue a warning. def warn_on_unexported_main(symbolses): @@ -1333,6 +1341,9 @@ def add_library(lib): force_include.add('libasan_rt_wasm') add_library(system_libs_map['libasan_rt_wasm']) + if shared.Settings.WASI: + add_library(system_libs_map['libwasi']) + libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. # libc++abi and libc++ *static* linking is tricky. e.g. cxa_demangle.cpp disables c++ From 479e297994095d0979e462c9ab7018f90dbef080 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Sat, 7 Sep 2019 19:03:07 -0700 Subject: [PATCH 02/20] fixups [ci skip] --- emcc.py | 9 +++++- src/settings.js | 26 ++++++++++++++-- system/lib/wasi.c | 77 +++-------------------------------------------- tools/shared.py | 7 +++-- 4 files changed, 40 insertions(+), 79 deletions(-) diff --git a/emcc.py b/emcc.py index bd81adfd31f8e..87441d06ea021 100755 --- a/emcc.py +++ b/emcc.py @@ -1729,7 +1729,14 @@ def check_human_readable_list(items): options.separate_asm = True if shared.Settings.WASI: - # we cannot legalize the JS FFI when using wasi, as it can run standalone in wasm + if not shared.Settings.WASM_BACKEND: + exit_with_error('wasi is only available in the upstream wasm backend path') + if shared.Settings.USE_PTHREADS: + exit_with_error('wasi does not support pthreads yet') + if shared.Settings.SIMD: + exit_with_error('wasi does not support simd yet') + # the wasm must be runnable without the JS, so there cannot be anything that + # requires JS legalization shared.Settings.LEGALIZE_JS_FFI = 0 if shared.Settings.WASM_BACKEND: diff --git a/src/settings.js b/src/settings.js index 8398749d649f4..486d1d0c3e029 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1072,9 +1072,29 @@ var USE_GLFW = 2; // port, which can useful for local dev work on binaryen itself). var WASM = 1; -// Whether to use the WASI APIs. This still emits a JS+wasm combination, but -// the wasm part can run as a wasi executable inside a wasi runtime, without -// any JS - basically, the JS is just a convenient way to run it on the Web. +// Whether to emit a wasm file with maximum WASI compatibility. Note that +// even without this option we use some wasi syscalls in our ABI, which +// makes sense to do when the choice is between something arbitrary (like +// linux/musl or something else) or wasi. However, there are cases where +// the choice is not arbitrary, and supporting wasi comes at a cost, and +// that is gated behind this option. +// +// For example, when this option is set then we assume the WASI view of +// the world in which the wasm is standalone - we cannot depend on JS for +// anything. That means, for example, that we don't import the memory, +// since we must create it ourselves. And that prevents various optimizations +// like preloading files into memory while the wasm is still loading (which +// is why Emscripten by default creates the Memory in JS). +// +// Note that this option is a best-effort: we enable all wasi compatibility +// we have, but if you ask for non-wasi APIs, we will still use them, and +// if you try to run in a wasi VM you'll get an error. Please file an issue +// if you find an API we missed and need to add, but in general most such +// APIs may just be missing in wasi. +// +// When this option is set we still emit both JS and wasm. The wasm will +// be runnable in a wasi VM, and you can also run it using the JS in a +// JS+wasm VM - basically, the JS is a convenient way to run it. var WASI = 0; // Whether to use the WebAssembly backend that is in development in LLVM. You diff --git a/system/lib/wasi.c b/system/lib/wasi.c index b18d66e2a3445..1685e3d953492 100644 --- a/system/lib/wasi.c +++ b/system/lib/wasi.c @@ -3,10 +3,6 @@ * Emscripten is available under two separate licenses, the MIT license and the * University of Illinois/NCSA Open Source License. Both these licenses can be * found in the LICENSE file. - * - * C++ exception handling support stubs. This is included when exception - * throwing is disabled - so no exceptions should exist at all. If the code still - * uses them, these stubs will throw at runtime. */ #include @@ -16,7 +12,11 @@ #include "wasi.h" -// WASI APIs +/* + * WASI support code. These are compiled with the program, and call out + * using wasi APIs, which can be provided either by a wasi VM or by our + * emitted JS. + */ // libc @@ -55,70 +55,3 @@ void *emscripten_memcpy_big(void *restrict dest, const void *restrict src, size_ } return dest; } - -// I/O syscalls - we support printf etc., but no filesystem access for now. - -static intptr_t _vararg; - -static void set_vararg(intptr_t vararg) { - _vararg = vararg; -} - -static int get_vararg_i32() { - int ret = *(int*)_vararg; - _vararg += 4; - return ret; -} - -int __syscall6(int id, intptr_t vararg) { // close - return 0; -} - -int __syscall54(int id, intptr_t vararg) { // ioctl - return 0; -} - -int __syscall140(int id, intptr_t vararg) { // llseek - return 0; -} - -int __syscall146(int id, intptr_t vararg) { // writev - set_vararg(vararg); - // hack to support printf, similar to library_syscalls.js handling of SYSCALLS_REQUIRE_FILESYSTEM=0 - int stream = get_vararg_i32(); - // Luckily iovs are identical in musl and wasi - struct __wasi_ciovec_t* iov = (struct __wasi_ciovec_t*)get_vararg_i32(); - size_t iovcnt = get_vararg_i32(); - size_t ret = 0; - for (int i = 0; i < iovcnt; i++) { - size_t num; - int result = __wasi_fd_write(stream, iov, 1, &num); - // TODO: error handling - ret += num; - iov++; - } - return ret; -} - -// Memory management - -extern intptr_t emscripten_get_sbrk_ptr(); - -static const size_t WASM_PAGE_SIZE = 65536; - -void *sbrk(intptr_t increment) { - intptr_t old_brk = emscripten_get_sbrk_ptr(); - // TODO: overflow checks - intptr_t updated_brk = old_brk + increment; - uintptr_t old_size = __builtin_wasm_memory_size(0) * WASM_PAGE_SIZE; - if (updated_brk >= old_size) { - // Try to grow memory. - intptr_t diff = updated_brk - old_size; - uintptr_t result = __builtin_wasm_memory_grow(0, (diff + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE); - if (result == SIZE_MAX) { - errno = ENOMEM; - return (void*)-1; - } - } - return (void*)old_brk; -} diff --git a/tools/shared.py b/tools/shared.py index dce4eda88089e..3b1cf485af1fd 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1796,8 +1796,9 @@ def link_lld(args, target, opts=[], lto_level=0): '--lto-O%d' % lto_level, ] + args - # wasi does not import the memory, but for JS it is convenient to do so, - # as it allows us to set up memory even before the wasm module arrives + # wasi does not import the memory (but for JS it is efficient to do so, + # as it allows us to set up memory, preload files, etc. even before the + # wasm module arrives) if not Settings.WASI: cmd.append('--import-memory') cmd.append('--import-table') @@ -2469,7 +2470,7 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - # For wasi, in theory we could minify the non-wasi ones, but the point of wasi is to not have such things anyhow. + # For wasi, we must not minify the wasi ones (and there should be no others), so disable minification. if Settings.EMITTING_JS and not Settings.AUTODEBUG and not Settings.WASI: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file From b25ee3987b96cb420e7be1aa5292afee34d74cd6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Sep 2019 15:58:46 -0700 Subject: [PATCH 03/20] PURE_WASM --- emcc.py | 8 +++--- src/modules.js | 2 +- src/preamble.js | 5 ++-- src/settings.js | 40 ++++++++++++++---------------- system/lib/{wasi.c => pure_wasm.c} | 3 ++- tools/shared.py | 9 ++++--- tools/system_libs.py | 8 +++--- 7 files changed, 39 insertions(+), 36 deletions(-) rename system/lib/{wasi.c => pure_wasm.c} (92%) diff --git a/emcc.py b/emcc.py index 671b727b01f16..694e17e30a479 100755 --- a/emcc.py +++ b/emcc.py @@ -1742,13 +1742,13 @@ def check_human_readable_list(items): if shared.Settings.MINIMAL_RUNTIME and not shared.Settings.WASM: options.separate_asm = True - if shared.Settings.WASI: + if shared.Settings.PURE_WASM: if not shared.Settings.WASM_BACKEND: - exit_with_error('wasi is only available in the upstream wasm backend path') + exit_with_error('PURE_WASM is only available in the upstream wasm backend path') if shared.Settings.USE_PTHREADS: - exit_with_error('wasi does not support pthreads yet') + exit_with_error('PURE_WASM does not support pthreads yet') if shared.Settings.SIMD: - exit_with_error('wasi does not support simd yet') + exit_with_error('PURE_WASM does not support simd yet') # the wasm must be runnable without the JS, so there cannot be anything that # requires JS legalization shared.Settings.LEGALIZE_JS_FFI = 0 diff --git a/src/modules.js b/src/modules.js index 719d919d74c46..0693f10baafc3 100644 --- a/src/modules.js +++ b/src/modules.js @@ -152,7 +152,7 @@ var LibraryManager = { libraries.push('library_glemu.js'); } - if (WASI) { + if (PURE_WASM) { libraries.push('library_wasi.js'); } diff --git a/src/preamble.js b/src/preamble.js index 6ef1c28ef1ede..b584ce01cbb73 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1069,8 +1069,9 @@ function createWasm(env) { exports = Asyncify.instrumentWasmExports(exports); #endif Module['asm'] = exports; -#if WASI - // In wasi the memory is exported. +#if PURE_WASM + // In pure wasm mode the memory is created in the wasm (not imported), and + // then exported. updateGlobalBufferAndViews(exports['memory'].buffer); #endif #if USE_PTHREADS diff --git a/src/settings.js b/src/settings.js index 7a878994255c0..924e3e49fc176 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1072,30 +1072,28 @@ var USE_GLFW = 2; // port, which can useful for local dev work on binaryen itself). var WASM = 1; -// Whether to emit a wasm file with maximum WASI compatibility. Note that -// even without this option we use some wasi syscalls in our ABI, which -// makes sense to do when the choice is between something arbitrary (like -// linux/musl or something else) or wasi. However, there are cases where -// the choice is not arbitrary, and supporting wasi comes at a cost, and -// that is gated behind this option. +// PURE_WASM indicates that we want to emit a wasm file that can run without +// JavaScript. The file will use standard APIs such as wasi as much as possible +// to achieve that. // -// For example, when this option is set then we assume the WASI view of -// the world in which the wasm is standalone - we cannot depend on JS for -// anything. That means, for example, that we don't import the memory, -// since we must create it ourselves. And that prevents various optimizations -// like preloading files into memory while the wasm is still loading (which -// is why Emscripten by default creates the Memory in JS). +// This option does not guarantee that the wasm can be used by itself - if you +// use APIs with no non-JS alternative, we will still use those (e.g., WebGL +// at the time of writing this). // -// Note that this option is a best-effort: we enable all wasi compatibility -// we have, but if you ask for non-wasi APIs, we will still use them, and -// if you try to run in a wasi VM you'll get an error. Please file an issue -// if you find an API we missed and need to add, but in general most such -// APIs may just be missing in wasi. +// We may still emit JS with this flag, but the JS should only be a convenient +// way to run the wasm on the Web or in Node.js, and you can run the wasm by +// itself without that JS (again, unless you use APIs for which there is no +// non-JS alternative). // -// When this option is set we still emit both JS and wasm. The wasm will -// be runnable in a wasi VM, and you can also run it using the JS in a -// JS+wasm VM - basically, the JS is a convenient way to run it. -var WASI = 0; +// Note that even without this option we try to use wasi etc. syscalls as much +// as possible. What this option changes is that we do so even when it means +// a tradeoff with JS size. For example, when this option is set we do not +// import the Memory - importing it is useful for JS, so that JS can start to +// use it before the wasm is even loaded, but in wasi and other wasm-only +// environments the expectation is to create the memory in the wasm itself. +// Doing so prevents some possible JS optimizations, so we only do it behind +// this flag. +var PURE_WASM = 0; // Whether to use the WebAssembly backend that is in development in LLVM. You // should not set this yourself, instead set EMCC_WASM_BACKEND=1 in the diff --git a/system/lib/wasi.c b/system/lib/pure_wasm.c similarity index 92% rename from system/lib/wasi.c rename to system/lib/pure_wasm.c index 1685e3d953492..66b747681bce4 100644 --- a/system/lib/wasi.c +++ b/system/lib/pure_wasm.c @@ -41,7 +41,8 @@ void *emscripten_memcpy_big(void *restrict dest, const void *restrict src, size_ // This normally calls out into JS which can do a single fast operation, // but with wasi we can't do that. As this is called when n >= 8192, we // can just split into smaller calls. - // TODO optimize, maybe build our memcpy with a wasi variant? + // TODO optimize, maybe build our memcpy with a wasi variant, maybe have + // a SIMD variant, etc. const int CHUNK = 8192; unsigned char* d = (unsigned char*)dest; unsigned char* s = (unsigned char*)src; diff --git a/tools/shared.py b/tools/shared.py index 58c4b5b82a024..0a06287b7d280 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1825,7 +1825,7 @@ def link_lld(args, target, opts=[], lto_level=0): # wasi does not import the memory (but for JS it is efficient to do so, # as it allows us to set up memory, preload files, etc. even before the # wasm module arrives) - if not Settings.WASI: + if not Settings.PURE_WASM: cmd.append('--import-memory') cmd.append('--import-table') @@ -2496,7 +2496,7 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if Settings.EMITTING_JS and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.WASI: + if Settings.EMITTING_JS and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.PURE_WASM: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file @@ -2532,7 +2532,10 @@ def metadce(js_file, wasm_file, minify_whitespace, debug_info): if export in Building.user_requested_exports or Settings.EXPORT_ALL: item['root'] = True # fix wasi imports TODO: support wasm stable with an option? - WASI_IMPORTS = set(['fd_write']) + WASI_IMPORTS = set([ + 'fd_write', + 'proc_exit', + ]) for item in graph: if 'import' in item and item['import'][1][1:] in WASI_IMPORTS: item['import'][0] = 'wasi_unstable' diff --git a/tools/system_libs.py b/tools/system_libs.py index 02d5a3ce79627..7cc63ab3f366c 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1140,12 +1140,12 @@ class libasan_rt_wasm(SanitizerLibrary): src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'asan'] -class libwasi(WasiLibrary): +class libpurewasm(Library): name = 'libwasi' cflags = ['-Os'] src_dir = ['system', 'lib'] - src_files = ['wasi.c'] + src_files = ['pure_wasm.c'] # If main() is not in EXPORTED_FUNCTIONS, it may be dce'd out. This can be @@ -1324,8 +1324,8 @@ def add_library(lib): force_include.add('libasan_rt_wasm') add_library(system_libs_map['libasan_rt_wasm']) - if shared.Settings.WASI: - add_library(system_libs_map['libwasi']) + if shared.Settings.PURE_WASM: + add_library(system_libs_map['libpurewasm']) libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. From fab6ceb10847ec9adf57420da233378962619f3c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Sep 2019 16:16:11 -0700 Subject: [PATCH 04/20] wip [ci skip] --- ChangeLog.md | 4 ++++ emcc.py | 2 +- site/source/docs/tools_reference/emcc.rst | 2 +- src/settings.js | 3 --- system/lib/pure_wasm.c | 2 +- tests/test_other.py | 24 +++++++++++++++-------- tools/shared.py | 4 ++-- tools/system_libs.py | 2 +- 8 files changed, 26 insertions(+), 17 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 1f82a1ad971ee..cd18404106531 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -18,6 +18,10 @@ See docs/process.md for how version tagging works. Current Trunk ------------- + - Remove `EMITTING_JS` flag, and replace it with `PURE_WASM`. That flag indicates + that we want the wasm to be as standalone as possible. We may still emit JS in + that case, but the JS would just be a convenient way to run the wasm on the Web + or in Node.js. v.1.38.44: 09/11/2019 --------------------- diff --git a/emcc.py b/emcc.py index 694e17e30a479..9e9081df7be57 100755 --- a/emcc.py +++ b/emcc.py @@ -1677,7 +1677,7 @@ def check_human_readable_list(items): # wasm outputs are only possible with a side wasm if target.endswith(WASM_ENDINGS): - shared.Settings.EMITTING_JS = 0 + shared.Settings.PURE_WASM = 1 js_target = misc_temp_files.get(suffix='.js').name if shared.Settings.EVAL_CTORS: diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index 3439e226dd565..485d9e70f6403 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -443,7 +443,7 @@ Options that are modified or new in *emcc* are listed below: - **.html** : HTML + separate JavaScript file (**.js**; + separate **.wasm** file if emitting WebAssembly). - **.bc** : LLVM bitcode. - **.o** : LLVM bitcode (same as .bc), unless in `WASM_OBJECT_FILES` mode, in which case it will contain a WebAssembly object. - - **.wasm** : WebAssembly without JavaScript support code ("standalone wasm"). + - **.wasm** : WebAssembly without JavaScript support code ("standalone wasm"; this enables ``PURE_WASM``). .. note:: If ``--memory-init-file`` is used, a **.mem** file will be created in addition to the generated **.js** and/or **.html** file. diff --git a/src/settings.js b/src/settings.js index 924e3e49fc176..454e39523dd06 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1049,9 +1049,6 @@ var EMTERPRETIFY_SYNCLIST = []; // whether js opts will be run, after the main compiler var RUNNING_JS_OPTS = 0; -// whether we are emitting JS glue code -var EMITTING_JS = 1; - // whether we are in the generate struct_info bootstrap phase var BOOTSTRAPPING_STRUCT_INFO = 0; diff --git a/system/lib/pure_wasm.c b/system/lib/pure_wasm.c index 66b747681bce4..32f24bccca4bf 100644 --- a/system/lib/pure_wasm.c +++ b/system/lib/pure_wasm.c @@ -10,7 +10,7 @@ #include #include -#include "wasi.h" +#include /* * WASI support code. These are compiled with the program, and call out diff --git a/tests/test_other.py b/tests/test_other.py index 42cba1f7e3943..a84f300870ac5 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8309,12 +8309,16 @@ def run(args, expected): run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1'], (2 * 1024 * 1024 * 1024 - 65536) // 16384) run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1', '-s', 'WASM_MEM_MAX=128MB'], 2048 * 4) - def test_wasm_targets(self): + @no_fastcomp('only upstream supports PURE_WASM') + def test_wasm_target_and_PURE_WASM(self): + # PURE_WASM means we never minify imports and exports. That flag is implied + # when we do -o X.wasm (i.e. do not emit any JS). for opts, potentially_expect_minified_exports_and_imports in ( - ([], False), - (['-O2'], False), - (['-O3'], True), - (['-Os'], True), + ([], False), + (['-O2'], False), + (['-O3'], True), + (['-O3', '-s', 'PURE_WASM'], False), + (['-Os'], True), ): for target in ('out.js', 'out.wasm'): expect_minified_exports_and_imports = potentially_expect_minified_exports_and_imports and target.endswith('.js') @@ -8330,13 +8334,17 @@ def test_wasm_targets(self): exports = [line.strip().split(' ')[1].replace('"', '') for line in wast_lines if "(export " in line] imports = [line.strip().split(' ')[2].replace('"', '') for line in wast_lines if "(import " in line] exports_and_imports = exports + imports - print(exports) - print(imports) + print('exports', exports) + print('imports', imports) if expect_minified_exports_and_imports: assert 'a' in exports_and_imports else: assert 'a' not in exports_and_imports - assert 'memory' in exports_and_imports, 'some things are not minified anyhow' + assert 'memory' in exports_and_imports or 'fd_write' in exports_and_imports, 'some things are not minified anyhow' + # verify the wasm runs with the JS + if target.endswith('.js'): + self.assertContained('hello, world!', run_js('out.js')) + # TODO: verify the wasm runs in a wasi VM def test_wasm_targets_side_module(self): # side modules do allow a wasm target diff --git a/tools/shared.py b/tools/shared.py index 0a06287b7d280..5de49d5a4434e 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2492,11 +2492,11 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac passes.append('minifyWhitespace') logger.debug('running post-meta-DCE cleanup on shell code: ' + ' '.join(passes)) js_file = Building.acorn_optimizer(js_file, passes) - # also minify the names used between js and wasm, if we emitting JS (then the JS knows how to load the minified names) + # Also minify the names used between js and wasm, if we are emitting an optimized JS+wasm combo (then the JS knows how to load the minified names). # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if Settings.EMITTING_JS and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.PURE_WASM: + if not Settings.PURE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file diff --git a/tools/system_libs.py b/tools/system_libs.py index 7cc63ab3f366c..d22c6a4f411ec 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1141,7 +1141,7 @@ class libasan_rt_wasm(SanitizerLibrary): class libpurewasm(Library): - name = 'libwasi' + name = 'libpurewasm' cflags = ['-Os'] src_dir = ['system', 'lib'] From 664e17bc964dd0aca99b3310fbeca4931a9cf60f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Sep 2019 16:28:00 -0700 Subject: [PATCH 05/20] fixes [ci skip] --- emscripten.py | 2 ++ tests/test_other.py | 2 +- tools/shared.py | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/emscripten.py b/emscripten.py index 6564ba80fb893..2ca933d2d9b6f 100644 --- a/emscripten.py +++ b/emscripten.py @@ -2316,6 +2316,8 @@ def debug_copy(src, dst): cmd.append('--global-base=%s' % shared.Settings.GLOBAL_BASE) if shared.Settings.SAFE_STACK: cmd.append('--check-stack-overflow') + if shared.Settings.PURE_WASM: + cmd.append('--pure-wasm') shared.print_compiler_stage(cmd) stdout = shared.check_call(cmd, stdout=subprocess.PIPE).stdout if write_source_map: diff --git a/tests/test_other.py b/tests/test_other.py index a84f300870ac5..2f6aebf575cbf 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8325,7 +8325,7 @@ def test_wasm_target_and_PURE_WASM(self): print(opts, potentially_expect_minified_exports_and_imports, target, ' => ', expect_minified_exports_and_imports) self.clear() - run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', target] + opts) + run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', target, '--profiling'] + opts) self.assertExists('out.wasm') if target.endswith('.wasm'): assert not os.path.exists('out.js'), 'only wasm requested' diff --git a/tools/shared.py b/tools/shared.py index 5de49d5a4434e..5a38751ca5177 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2531,6 +2531,14 @@ def metadce(js_file, wasm_file, minify_whitespace, debug_info): export = '_' + export if export in Building.user_requested_exports or Settings.EXPORT_ALL: item['root'] = True + # in pure wasm, always export the memory + if Settings.PURE_WASM: + graph.append({ + 'export': 'memory', + 'name': 'emcc$export$memory', + 'reaches': [], + 'root': True + }) # fix wasi imports TODO: support wasm stable with an option? WASI_IMPORTS = set([ 'fd_write', From 40e2f5821ac99095c8d9b89797a42a7104cc253a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Sep 2019 16:47:12 -0700 Subject: [PATCH 06/20] more [ci skip] --- .circleci/config.yml | 4 ++++ emcc.py | 2 ++ system/lib/pure_wasm.c | 9 +++++++++ tests/test_other.py | 12 +++++++++--- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 937c1820893ad..5fa7f229030c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -419,6 +419,10 @@ jobs: test-upstream-other: executor: bionic steps: + - run: + name: get wasmer + command: | + curl https://get.wasmer.io -sSfL | sh - run-tests: # see explanations in the fastcomp skips for these, earlier test_targets: "other skip:other.test_native_link_error_message skip:other.test_emcc_v" diff --git a/emcc.py b/emcc.py index 9e9081df7be57..6169c2c3c3426 100755 --- a/emcc.py +++ b/emcc.py @@ -1749,6 +1749,8 @@ def check_human_readable_list(items): exit_with_error('PURE_WASM does not support pthreads yet') if shared.Settings.SIMD: exit_with_error('PURE_WASM does not support simd yet') + if shared.Settings.ALLOW_MEMORY_GROWTH: + exit_with_error('PURE_WASM does not support memory growth yet') # the wasm must be runnable without the JS, so there cannot be anything that # requires JS legalization shared.Settings.LEGALIZE_JS_FFI = 0 diff --git a/system/lib/pure_wasm.c b/system/lib/pure_wasm.c index 32f24bccca4bf..10895ed288f33 100644 --- a/system/lib/pure_wasm.c +++ b/system/lib/pure_wasm.c @@ -56,3 +56,12 @@ void *emscripten_memcpy_big(void *restrict dest, const void *restrict src, size_ } return dest; } + +static const int WASM_PAGE_SIZE = 65536; + +// Note that this does not support memory growth in JS because we don't update the JS +// heaps. Wasm and wasi lack a good API for that. +int emscripten_resize_heap(size_t size) { + size_t result = __builtin_wasm_memory_grow(0, (size + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE); + return result != (size_t)-1; +} diff --git a/tests/test_other.py b/tests/test_other.py index 2f6aebf575cbf..24d5c158c1130 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -30,7 +30,7 @@ raise Exception('do not run this file directly; do something like: tests/runner.py other') from tools.shared import Building, PIPE, run_js, run_process, STDOUT, try_delete, listify -from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, MACOS, LLVM_ROOT, EMCONFIG, EM_BUILD_VERBOSE +from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, MACOS, LINUX, LLVM_ROOT, EMCONFIG, EM_BUILD_VERBOSE from tools.shared import CLANG, CLANG_CC, CLANG_CPP, LLVM_AR from tools.shared import COMPILER_ENGINE, NODE_JS, SPIDERMONKEY_ENGINE, JS_ENGINES, V8_ENGINE from tools.shared import WebAssembly @@ -8325,7 +8325,7 @@ def test_wasm_target_and_PURE_WASM(self): print(opts, potentially_expect_minified_exports_and_imports, target, ' => ', expect_minified_exports_and_imports) self.clear() - run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', target, '--profiling'] + opts) + run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', target] + opts) self.assertExists('out.wasm') if target.endswith('.wasm'): assert not os.path.exists('out.js'), 'only wasm requested' @@ -8344,7 +8344,13 @@ def test_wasm_target_and_PURE_WASM(self): # verify the wasm runs with the JS if target.endswith('.js'): self.assertContained('hello, world!', run_js('out.js')) - # TODO: verify the wasm runs in a wasi VM + # verify the wasm runs in a wasm VM, without the JS + if LINUX: # TODO: other platforms + if target.endswith('.wasm') or 'PURE_WASM' in opts: + print('running pure wasm') + WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) + out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) def test_wasm_targets_side_module(self): # side modules do allow a wasm target From 90b1eaa3a70b3d1f75a8999964508cee70fc6168 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Sep 2019 16:52:35 -0700 Subject: [PATCH 07/20] comments [ci skip] --- src/preamble.js | 1 + src/settings.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/preamble.js b/src/preamble.js index b584ce01cbb73..bf3d936e49804 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1072,6 +1072,7 @@ function createWasm(env) { #if PURE_WASM // In pure wasm mode the memory is created in the wasm (not imported), and // then exported. + // TODO: do not create a Memory earlier in JS updateGlobalBufferAndViews(exports['memory'].buffer); #endif #if USE_PTHREADS diff --git a/src/settings.js b/src/settings.js index 454e39523dd06..be268228a1c66 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1080,7 +1080,7 @@ var WASM = 1; // We may still emit JS with this flag, but the JS should only be a convenient // way to run the wasm on the Web or in Node.js, and you can run the wasm by // itself without that JS (again, unless you use APIs for which there is no -// non-JS alternative). +// non-JS alternative) in a wasm runtime like wasmer or wasmtime. // // Note that even without this option we try to use wasi etc. syscalls as much // as possible. What this option changes is that we do so even when it means From 78db28a2b91377fe26417b5e5c77993ec9bf02e2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 13:16:13 -0700 Subject: [PATCH 08/20] standalone [ci skip] --- ChangeLog.md | 2 +- emcc.py | 12 ++++++------ emscripten.py | 4 ++-- site/source/docs/tools_reference/emcc.rst | 2 +- src/modules.js | 2 +- src/preamble.js | 2 +- src/settings.js | 4 ++-- system/lib/{pure_wasm.c => standalone_wasm.c} | 0 tests/test_other.py | 18 +++++++++--------- tools/shared.py | 8 ++++---- tools/system_libs.py | 10 +++++----- 11 files changed, 32 insertions(+), 32 deletions(-) rename system/lib/{pure_wasm.c => standalone_wasm.c} (100%) diff --git a/ChangeLog.md b/ChangeLog.md index cd18404106531..25a22e5015183 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -18,7 +18,7 @@ See docs/process.md for how version tagging works. Current Trunk ------------- - - Remove `EMITTING_JS` flag, and replace it with `PURE_WASM`. That flag indicates + - Remove `EMITTING_JS` flag, and replace it with `STANDALNE_WASM`. That flag indicates that we want the wasm to be as standalone as possible. We may still emit JS in that case, but the JS would just be a convenient way to run the wasm on the Web or in Node.js. diff --git a/emcc.py b/emcc.py index 6169c2c3c3426..200c7059046bd 100755 --- a/emcc.py +++ b/emcc.py @@ -1677,7 +1677,7 @@ def check_human_readable_list(items): # wasm outputs are only possible with a side wasm if target.endswith(WASM_ENDINGS): - shared.Settings.PURE_WASM = 1 + shared.Settings.STANDALONE_WASM = 1 js_target = misc_temp_files.get(suffix='.js').name if shared.Settings.EVAL_CTORS: @@ -1742,15 +1742,15 @@ def check_human_readable_list(items): if shared.Settings.MINIMAL_RUNTIME and not shared.Settings.WASM: options.separate_asm = True - if shared.Settings.PURE_WASM: + if shared.Settings.STANDALONE_WASM: if not shared.Settings.WASM_BACKEND: - exit_with_error('PURE_WASM is only available in the upstream wasm backend path') + exit_with_error('STANDALONE_WASM is only available in the upstream wasm backend path') if shared.Settings.USE_PTHREADS: - exit_with_error('PURE_WASM does not support pthreads yet') + exit_with_error('STANDALONE_WASM does not support pthreads yet') if shared.Settings.SIMD: - exit_with_error('PURE_WASM does not support simd yet') + exit_with_error('STANDALONE_WASM does not support simd yet') if shared.Settings.ALLOW_MEMORY_GROWTH: - exit_with_error('PURE_WASM does not support memory growth yet') + exit_with_error('STANDALONE_WASM does not support memory growth yet') # the wasm must be runnable without the JS, so there cannot be anything that # requires JS legalization shared.Settings.LEGALIZE_JS_FFI = 0 diff --git a/emscripten.py b/emscripten.py index 2ca933d2d9b6f..bfce56d0dce67 100644 --- a/emscripten.py +++ b/emscripten.py @@ -2316,8 +2316,8 @@ def debug_copy(src, dst): cmd.append('--global-base=%s' % shared.Settings.GLOBAL_BASE) if shared.Settings.SAFE_STACK: cmd.append('--check-stack-overflow') - if shared.Settings.PURE_WASM: - cmd.append('--pure-wasm') + if shared.Settings.STANDALONE_WASM: + cmd.append('--standalone-wasm') shared.print_compiler_stage(cmd) stdout = shared.check_call(cmd, stdout=subprocess.PIPE).stdout if write_source_map: diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index 485d9e70f6403..d36115a29c13e 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -443,7 +443,7 @@ Options that are modified or new in *emcc* are listed below: - **.html** : HTML + separate JavaScript file (**.js**; + separate **.wasm** file if emitting WebAssembly). - **.bc** : LLVM bitcode. - **.o** : LLVM bitcode (same as .bc), unless in `WASM_OBJECT_FILES` mode, in which case it will contain a WebAssembly object. - - **.wasm** : WebAssembly without JavaScript support code ("standalone wasm"; this enables ``PURE_WASM``). + - **.wasm** : WebAssembly without JavaScript support code ("standalone wasm"; this enables ``STANDALONE_WASM``). .. note:: If ``--memory-init-file`` is used, a **.mem** file will be created in addition to the generated **.js** and/or **.html** file. diff --git a/src/modules.js b/src/modules.js index 0693f10baafc3..2fad1837b593e 100644 --- a/src/modules.js +++ b/src/modules.js @@ -152,7 +152,7 @@ var LibraryManager = { libraries.push('library_glemu.js'); } - if (PURE_WASM) { + if (STANDALONE_WASM) { libraries.push('library_wasi.js'); } diff --git a/src/preamble.js b/src/preamble.js index bf3d936e49804..14c3f9536f9aa 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1069,7 +1069,7 @@ function createWasm(env) { exports = Asyncify.instrumentWasmExports(exports); #endif Module['asm'] = exports; -#if PURE_WASM +#if STANDALONE_WASM // In pure wasm mode the memory is created in the wasm (not imported), and // then exported. // TODO: do not create a Memory earlier in JS diff --git a/src/settings.js b/src/settings.js index be268228a1c66..8575f2b773b4d 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1069,7 +1069,7 @@ var USE_GLFW = 2; // port, which can useful for local dev work on binaryen itself). var WASM = 1; -// PURE_WASM indicates that we want to emit a wasm file that can run without +// STANDALONE indicates that we want to emit a wasm file that can run without // JavaScript. The file will use standard APIs such as wasi as much as possible // to achieve that. // @@ -1090,7 +1090,7 @@ var WASM = 1; // environments the expectation is to create the memory in the wasm itself. // Doing so prevents some possible JS optimizations, so we only do it behind // this flag. -var PURE_WASM = 0; +var STANDALONE_WASM = 0; // Whether to use the WebAssembly backend that is in development in LLVM. You // should not set this yourself, instead set EMCC_WASM_BACKEND=1 in the diff --git a/system/lib/pure_wasm.c b/system/lib/standalone_wasm.c similarity index 100% rename from system/lib/pure_wasm.c rename to system/lib/standalone_wasm.c diff --git a/tests/test_other.py b/tests/test_other.py index 24d5c158c1130..0c774bc6c0777 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8309,16 +8309,16 @@ def run(args, expected): run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1'], (2 * 1024 * 1024 * 1024 - 65536) // 16384) run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1', '-s', 'WASM_MEM_MAX=128MB'], 2048 * 4) - @no_fastcomp('only upstream supports PURE_WASM') - def test_wasm_target_and_PURE_WASM(self): - # PURE_WASM means we never minify imports and exports. That flag is implied + @no_fastcomp('only upstream supports STANDALONE_WASM') + def test_wasm_target_and_STANDALONE_WASM(self): + # STANDALONE_WASM means we never minify imports and exports. That flag is implied # when we do -o X.wasm (i.e. do not emit any JS). for opts, potentially_expect_minified_exports_and_imports in ( - ([], False), - (['-O2'], False), - (['-O3'], True), - (['-O3', '-s', 'PURE_WASM'], False), - (['-Os'], True), + ([], False), + (['-O2'], False), + (['-O3'], True), + (['-O3', '-s', 'STANDALONE_WASM'], False), + (['-Os'], True), ): for target in ('out.js', 'out.wasm'): expect_minified_exports_and_imports = potentially_expect_minified_exports_and_imports and target.endswith('.js') @@ -8346,7 +8346,7 @@ def test_wasm_target_and_PURE_WASM(self): self.assertContained('hello, world!', run_js('out.js')) # verify the wasm runs in a wasm VM, without the JS if LINUX: # TODO: other platforms - if target.endswith('.wasm') or 'PURE_WASM' in opts: + if target.endswith('.wasm') or 'STANDALONE_WASM' in opts: print('running pure wasm') WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout diff --git a/tools/shared.py b/tools/shared.py index 5a38751ca5177..08913438d4427 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1825,7 +1825,7 @@ def link_lld(args, target, opts=[], lto_level=0): # wasi does not import the memory (but for JS it is efficient to do so, # as it allows us to set up memory, preload files, etc. even before the # wasm module arrives) - if not Settings.PURE_WASM: + if not Settings.STANDALONE_WASM: cmd.append('--import-memory') cmd.append('--import-table') @@ -2496,7 +2496,7 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if not Settings.PURE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS: + if not Settings.STANDALONE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file @@ -2531,8 +2531,8 @@ def metadce(js_file, wasm_file, minify_whitespace, debug_info): export = '_' + export if export in Building.user_requested_exports or Settings.EXPORT_ALL: item['root'] = True - # in pure wasm, always export the memory - if Settings.PURE_WASM: + # in standalone wasm, always export the memory + if Settings.STANDALONE_WASM: graph.append({ 'export': 'memory', 'name': 'emcc$export$memory', diff --git a/tools/system_libs.py b/tools/system_libs.py index d22c6a4f411ec..3717150dde796 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1140,12 +1140,12 @@ class libasan_rt_wasm(SanitizerLibrary): src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'asan'] -class libpurewasm(Library): - name = 'libpurewasm' +class libstandalonewasm(Library): + name = 'libstandalonewasm' cflags = ['-Os'] src_dir = ['system', 'lib'] - src_files = ['pure_wasm.c'] + src_files = ['standalone_wasm.c'] # If main() is not in EXPORTED_FUNCTIONS, it may be dce'd out. This can be @@ -1324,8 +1324,8 @@ def add_library(lib): force_include.add('libasan_rt_wasm') add_library(system_libs_map['libasan_rt_wasm']) - if shared.Settings.PURE_WASM: - add_library(system_libs_map['libpurewasm']) + if shared.Settings.STANDALONE_WASM: + add_library(system_libs_map['libstandalonewasm']) libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. From 2e0770a91be464825e3be50032197e50ec3a75dd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 13:31:45 -0700 Subject: [PATCH 09/20] cleanup [ci skip] --- tests/test_other.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 0c774bc6c0777..71d2047911d9a 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8311,8 +8311,7 @@ def run(args, expected): @no_fastcomp('only upstream supports STANDALONE_WASM') def test_wasm_target_and_STANDALONE_WASM(self): - # STANDALONE_WASM means we never minify imports and exports. That flag is implied - # when we do -o X.wasm (i.e. do not emit any JS). + # STANDALONE_WASM means we never minify imports and exports. for opts, potentially_expect_minified_exports_and_imports in ( ([], False), (['-O2'], False), @@ -8320,9 +8319,11 @@ def test_wasm_target_and_STANDALONE_WASM(self): (['-O3', '-s', 'STANDALONE_WASM'], False), (['-Os'], True), ): + # targeting .wasm (without .js) means we enable STANDALONE_WASM automatically, and don't minify imports/exports for target in ('out.js', 'out.wasm'): expect_minified_exports_and_imports = potentially_expect_minified_exports_and_imports and target.endswith('.js') - print(opts, potentially_expect_minified_exports_and_imports, target, ' => ', expect_minified_exports_and_imports) + standalone = target.endswith('.wasm') or 'STANDALONE_WASM' in opts + print(opts, potentially_expect_minified_exports_and_imports, target, ' => ', expect_minified_exports_and_imports, standalone) self.clear() run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', target] + opts) @@ -8334,8 +8335,8 @@ def test_wasm_target_and_STANDALONE_WASM(self): exports = [line.strip().split(' ')[1].replace('"', '') for line in wast_lines if "(export " in line] imports = [line.strip().split(' ')[2].replace('"', '') for line in wast_lines if "(import " in line] exports_and_imports = exports + imports - print('exports', exports) - print('imports', imports) + print(' exports', exports) + print(' imports', imports) if expect_minified_exports_and_imports: assert 'a' in exports_and_imports else: @@ -8346,8 +8347,8 @@ def test_wasm_target_and_STANDALONE_WASM(self): self.assertContained('hello, world!', run_js('out.js')) # verify the wasm runs in a wasm VM, without the JS if LINUX: # TODO: other platforms - if target.endswith('.wasm') or 'STANDALONE_WASM' in opts: - print('running pure wasm') + if standalone: + print(' running in wasm runtime') WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout self.assertContained('hello, world!', out) From 1848451397eabb38103eb42ca41470cbbeb84173 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 15:10:31 -0700 Subject: [PATCH 10/20] wasmtime too [ci skip] --- .circleci/config.yml | 6 ++++++ tests/test_other.py | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fa7f229030c9..6546dda73f935 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -423,6 +423,12 @@ jobs: name: get wasmer command: | curl https://get.wasmer.io -sSfL | sh + - run: + name: get wasmtime + command: | + wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz + tar -xf wasmtime-dev-x86_64-linux.tar.xz + cp wasmtime-dev-x86_64-linux/wasmtime ~/ - run-tests: # see explanations in the fastcomp skips for these, earlier test_targets: "other skip:other.test_native_link_error_message skip:other.test_emcc_v" diff --git a/tests/test_other.py b/tests/test_other.py index 71d2047911d9a..0409c21e13960 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8348,10 +8348,20 @@ def test_wasm_target_and_STANDALONE_WASM(self): # verify the wasm runs in a wasm VM, without the JS if LINUX: # TODO: other platforms if standalone: - print(' running in wasm runtime') WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) - out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout - self.assertContained('hello, world!', out) + if os.path.isfile(WASMER): + print(' running in wasmer') + out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) + else: + print('[WARNING - no wasmer]') + WASMTIME = os.path.expanduser(os.path.join('~', 'wasmtime')) + if os.path.isfile(WASMTIME): + print(' running in wasmtime') + out = run_process([WASMTIME, 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) + else: + print('[WARNING - no wasmtime]') def test_wasm_targets_side_module(self): # side modules do allow a wasm target From f781f39a40b9424c8b452e0c92e03f9b934fea9e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 16:31:15 -0700 Subject: [PATCH 11/20] clear error on EMITTING_JS being removed --- src/settings.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/settings.js b/src/settings.js index 8575f2b773b4d..24cfa9dc762fc 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1648,4 +1648,5 @@ var LEGACY_SETTINGS = [ ['PRECISE_I64_MATH', [1, 2], 'Starting from Emscripten 1.38.26, PRECISE_I64_MATH is always enabled (https://github.com/emscripten-core/emscripten/pull/7935)'], ['MEMFS_APPEND_TO_TYPED_ARRAYS', [1], 'Starting from Emscripten 1.38.26, MEMFS_APPEND_TO_TYPED_ARRAYS=0 is no longer supported. MEMFS no longer supports using JS arrays for file data (https://github.com/emscripten-core/emscripten/pull/7918)'], ['ERROR_ON_MISSING_LIBRARIES', [1], 'missing libraries are always an error now'], + ['EMITTING_JS', [1], 'The new STANDALONE_WASM flag replaces this (replace EMITTING_JS=0 with STANDALONE_WASM=1)'], ]; From 5994a4c0d34be8a7357e5c2702c4f7e3f59013ce Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 16:34:58 -0700 Subject: [PATCH 12/20] text --- src/settings.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/settings.js b/src/settings.js index 24cfa9dc762fc..50e93b4ea3a8b 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1069,13 +1069,15 @@ var USE_GLFW = 2; // port, which can useful for local dev work on binaryen itself). var WASM = 1; -// STANDALONE indicates that we want to emit a wasm file that can run without +// STANDALONE_WASM indicates that we want to emit a wasm file that can run without // JavaScript. The file will use standard APIs such as wasi as much as possible // to achieve that. // // This option does not guarantee that the wasm can be used by itself - if you -// use APIs with no non-JS alternative, we will still use those (e.g., WebGL -// at the time of writing this). +// use APIs with no non-JS alternative, we will still use those (e.g., OpenGL +// at the time of writing this). This gives you the option to see which APIs +// are missing, and if you are compiling for a custom wasi embedding, to add +// those to your embedding. // // We may still emit JS with this flag, but the JS should only be a convenient // way to run the wasm on the Web or in Node.js, and you can run the wasm by From d129daffe04ed4d31a965d08f61fbef97c41b5c1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 16:46:35 -0700 Subject: [PATCH 13/20] standalone wasm library is upstream-only --- tools/system_libs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/system_libs.py b/tools/system_libs.py index 3717150dde796..4b6141421b6c3 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1147,6 +1147,9 @@ class libstandalonewasm(Library): src_dir = ['system', 'lib'] src_files = ['standalone_wasm.c'] + def can_build(self): + return shared.Settings.WASM_BACKEND + # If main() is not in EXPORTED_FUNCTIONS, it may be dce'd out. This can be # confusing, so issue a warning. From 5a0b65cc9d8d1fe28374eff69f97825e6abf343a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 16:57:24 -0700 Subject: [PATCH 14/20] here to there --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6546dda73f935..d51067ead24ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,6 +212,16 @@ jobs: name: freeze cache command: | echo "FROZEN_CACHE=True" >> ~/.emscripten + - run: + name: get wasmer + command: | + curl https://get.wasmer.io -sSfL | sh + - run: + name: get wasmtime + command: | + wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz + tar -xf wasmtime-dev-x86_64-linux.tar.xz + cp wasmtime-dev-x86_64-linux/wasmtime ~/ - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -419,16 +429,6 @@ jobs: test-upstream-other: executor: bionic steps: - - run: - name: get wasmer - command: | - curl https://get.wasmer.io -sSfL | sh - - run: - name: get wasmtime - command: | - wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz - tar -xf wasmtime-dev-x86_64-linux.tar.xz - cp wasmtime-dev-x86_64-linux/wasmtime ~/ - run-tests: # see explanations in the fastcomp skips for these, earlier test_targets: "other skip:other.test_native_link_error_message skip:other.test_emcc_v" From f89fa67c5c51a8e62c0485906315928cd7137672 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Sep 2019 16:59:37 -0700 Subject: [PATCH 15/20] here to there --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d51067ead24ee..ec7a8d9b63764 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,16 +212,6 @@ jobs: name: freeze cache command: | echo "FROZEN_CACHE=True" >> ~/.emscripten - - run: - name: get wasmer - command: | - curl https://get.wasmer.io -sSfL | sh - - run: - name: get wasmtime - command: | - wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz - tar -xf wasmtime-dev-x86_64-linux.tar.xz - cp wasmtime-dev-x86_64-linux/wasmtime ~/ - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -351,6 +341,16 @@ jobs: command: | apt-get update -q apt-get install -q -y python3 cmake + - run: + name: get wasmer + command: | + curl https://get.wasmer.io -sSfL | sh + - run: + name: get wasmtime + command: | + wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz + tar -xf wasmtime-dev-x86_64-linux.tar.xz + cp wasmtime-dev-x86_64-linux/wasmtime ~/ - run: name: install emsdk command: | From 998aa4a19fc4874e5efd2a657d98a1d400a595e1 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Wed, 18 Sep 2019 19:17:01 -0700 Subject: [PATCH 16/20] fixes --- emcc.py | 8 ++++++-- tools/shared.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/emcc.py b/emcc.py index 99d91805ba7d6..6e9f1ad8fbabe 100755 --- a/emcc.py +++ b/emcc.py @@ -1676,9 +1676,13 @@ def check_human_readable_list(items): if use_source_map(options): exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)') - # wasm outputs are only possible with a side wasm if target.endswith(WASM_ENDINGS): - shared.Settings.STANDALONE_WASM = 1 + # if the output is just a wasm file, it will normally be a standalone one, + # as there is no JS. an exception are side modules, as we can't tell at + # compile time whether JS will be involved or not - the main module may + # have JS, and the side module is expected to link against that. + if not shared.Settings.SIDE_MODULE: + shared.Settings.STANDALONE_WASM = 1 js_target = misc_temp_files.get(suffix='.js').name if shared.Settings.EVAL_CTORS: diff --git a/tools/shared.py b/tools/shared.py index 38edd1d45e72b..86f7badfeb56f 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2496,7 +2496,7 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if not Settings.STANDALONE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS: + if not Settings.STANDALONE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.SIDE_MODULE: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file From 521686d40b7407acbb25425bb739b39bb378f5e7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 19 Sep 2019 09:20:55 -0700 Subject: [PATCH 17/20] Update metadata, and add a metadata field for whether the wasm is standalone, as it does change the ABI --- tools/shared.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 86f7badfeb56f..9481ae075ffda 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -500,7 +500,7 @@ def get_emscripten_version(path): # NB: major version 0 implies no compatibility # NB: when changing the metadata format, we should only append new fields, not # reorder, modify, or remove existing ones. -(EMSCRIPTEN_METADATA_MAJOR, EMSCRIPTEN_METADATA_MINOR) = (0, 2) +(EMSCRIPTEN_METADATA_MAJOR, EMSCRIPTEN_METADATA_MINOR) = (0, 3) # For the JS/WASM ABI, specifies the minimum ABI version required of # the WASM runtime implementation by the generated WASM binary. It follows # semver and changes whenever C types change size/signedness or @@ -509,7 +509,7 @@ def get_emscripten_version(path): # change, increment EMSCRIPTEN_ABI_MINOR if EMSCRIPTEN_ABI_MAJOR == 0 # or the ABI change is backwards compatible, otherwise increment # EMSCRIPTEN_ABI_MAJOR and set EMSCRIPTEN_ABI_MINOR = 0. -(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 6) +(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 7) def generate_sanity(): @@ -3106,7 +3106,8 @@ def add_emscripten_metadata(js_file, wasm_file): WebAssembly.lebify(global_base) + WebAssembly.lebify(dynamic_base) + WebAssembly.lebify(dynamictop_ptr) + - WebAssembly.lebify(tempdouble_ptr) + WebAssembly.lebify(tempdouble_ptr) + + WebAssembly.lebify(int(Settings.STANDALONE_WASM)) # NB: more data can be appended here as long as you increase # the EMSCRIPTEN_METADATA_MINOR From c347034644b24e4a6c164e05b4e6901fb6666bd8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Sep 2019 12:39:45 -0700 Subject: [PATCH 18/20] fix fastcomp with -o X.wasm without SIDE_MODULE --- emcc.py | 6 ++++-- tests/test_other.py | 35 ++++++++++++++++++----------------- tools/shared.py | 4 ++-- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/emcc.py b/emcc.py index b82b0dd315dd5..98f50ebf0fc56 100755 --- a/emcc.py +++ b/emcc.py @@ -1679,7 +1679,8 @@ def check_human_readable_list(items): # as there is no JS. an exception are side modules, as we can't tell at # compile time whether JS will be involved or not - the main module may # have JS, and the side module is expected to link against that. - if not shared.Settings.SIDE_MODULE: + # we also do not support standalone mode in fastcomp. + if shared.Settings.WASM_BACKEND and not shared.Settings.SIDE_MODULE: shared.Settings.STANDALONE_WASM = 1 js_target = misc_temp_files.get(suffix='.js').name @@ -3006,7 +3007,8 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target, wasm_file=wasm_binary_target, expensive_optimizations=will_metadce(options), minify_whitespace=optimizer.minify_whitespace, - debug_info=intermediate_debug_info) + debug_info=intermediate_debug_info, + emitting_js=not target.endswith(WASM_ENDINGS)) save_intermediate_with_wasm('postclean', wasm_binary_target) def run_closure_compiler(final): diff --git a/tests/test_other.py b/tests/test_other.py index 1e85ec6e5e8c3..4a36dd32cc851 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8296,7 +8296,6 @@ def run(args, expected): run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1'], (2 * 1024 * 1024 * 1024 - 65536) // 16384) run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1', '-s', 'WASM_MEM_MAX=128MB'], 2048 * 4) - @no_fastcomp('only upstream supports STANDALONE_WASM') def test_wasm_target_and_STANDALONE_WASM(self): # STANDALONE_WASM means we never minify imports and exports. for opts, potentially_expect_minified_exports_and_imports in ( @@ -8306,6 +8305,8 @@ def test_wasm_target_and_STANDALONE_WASM(self): (['-O3', '-s', 'STANDALONE_WASM'], False), (['-Os'], True), ): + if 'STANDALONE_WASM' in opts and not self.is_wasm_backend(): + continue # targeting .wasm (without .js) means we enable STANDALONE_WASM automatically, and don't minify imports/exports for target in ('out.js', 'out.wasm'): expect_minified_exports_and_imports = potentially_expect_minified_exports_and_imports and target.endswith('.js') @@ -8333,22 +8334,22 @@ def test_wasm_target_and_STANDALONE_WASM(self): if target.endswith('.js'): self.assertContained('hello, world!', run_js('out.js')) # verify the wasm runs in a wasm VM, without the JS - if LINUX: # TODO: other platforms - if standalone: - WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) - if os.path.isfile(WASMER): - print(' running in wasmer') - out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout - self.assertContained('hello, world!', out) - else: - print('[WARNING - no wasmer]') - WASMTIME = os.path.expanduser(os.path.join('~', 'wasmtime')) - if os.path.isfile(WASMTIME): - print(' running in wasmtime') - out = run_process([WASMTIME, 'out.wasm'], stdout=PIPE).stdout - self.assertContained('hello, world!', out) - else: - print('[WARNING - no wasmtime]') + # TODO: more platforms than linux + if LINUX and standalone and self.is_wasm_backend(): + WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) + if os.path.isfile(WASMER): + print(' running in wasmer') + out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) + else: + print('[WARNING - no wasmer]') + WASMTIME = os.path.expanduser(os.path.join('~', 'wasmtime')) + if os.path.isfile(WASMTIME): + print(' running in wasmtime') + out = run_process([WASMTIME, 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) + else: + print('[WARNING - no wasmtime]') def test_wasm_targets_side_module(self): # side modules do allow a wasm target diff --git a/tools/shared.py b/tools/shared.py index 9481ae075ffda..271f5702425e3 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2466,7 +2466,7 @@ def closure_compiler(filename, pretty=True, advanced=True, extra_closure_args=[] # minify the final wasm+JS combination. this is done after all the JS # and wasm optimizations; here we do the very final optimizations on them @staticmethod - def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespace, debug_info): + def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespace, debug_info, emitting_js): # start with JSDCE, to clean up obvious JS garbage. When optimizing for size, # use AJSDCE (aggressive JS DCE, performs multiple iterations). Clean up # whitespace if necessary too. @@ -2496,7 +2496,7 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if not Settings.STANDALONE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.SIDE_MODULE: + if not Settings.STANDALONE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.SIDE_MODULE and emitting_js: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file From 67007b8275cd2be12f5d27e25668a1121937719c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Sep 2019 13:48:01 -0700 Subject: [PATCH 19/20] fix --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d372c18b6098..50d90cd2d498c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,16 +58,6 @@ commands: name: freeze cache command: | echo "FROZEN_CACHE=True" >> ~/.emscripten - - run: - name: get wasmer - command: | - curl https://get.wasmer.io -sSfL | sh - - run: - name: get wasmtime - command: | - wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz - tar -xf wasmtime-dev-x86_64-linux.tar.xz - cp wasmtime-dev-x86_64-linux/wasmtime ~/ - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -429,6 +419,16 @@ jobs: command: | apt-get update -q apt-get install -q -y python3 cmake + - run: + name: get wasmer + command: | + curl https://get.wasmer.io -sSfL | sh + - run: + name: get wasmtime + command: | + wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz + tar -xf wasmtime-dev-x86_64-linux.tar.xz + cp wasmtime-dev-x86_64-linux/wasmtime ~/ - checkout - build-upstream test-upstream-wasm0: From 0f6e911e9492f6e36e5699b1bc3dfa415d503ab3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Sep 2019 14:32:59 -0700 Subject: [PATCH 20/20] fix circleci maybe --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50d90cd2d498c..ce3e2fa2e76cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -419,6 +419,7 @@ jobs: command: | apt-get update -q apt-get install -q -y python3 cmake + - checkout - run: name: get wasmer command: | @@ -429,7 +430,6 @@ jobs: wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz tar -xf wasmtime-dev-x86_64-linux.tar.xz cp wasmtime-dev-x86_64-linux/wasmtime ~/ - - checkout - build-upstream test-upstream-wasm0: executor: bionic