Rename wasm2asm to wasm2js, emit ESM by default (#1642)

* Rename the `wasm2asm` tool to `wasm2js`

This commit performs a relatively simple rename of the `wasm2asm` tool to
`wasm2js`. The functionality of the tool doesn't change just yet but it's
intended that we'll start generating an ES module instead of just an `asm.js`
function soon.

* wasm2js: Support `*.wasm` input files

Previously `wasm2js` only supported `*.wast` files but to make it a bit easier
to use in tooling pipelines this commit adds support for reading in a `*.wasm`
file directly. Determining which parser to use depends on the input filename,
where the binary parser is used with `*.wasm` files and the wast parser is used
for all other files.

* wasm2js: Emit ESM imports/exports by default

This commit alters the default behavior of `wasm2js` to emit an ESM by default,
either importing items from the environment or exporting. Items like
initialization of memory are also handled here.

Author: alexcrichton

CMakeLists.txt

@@ -227,15 +227,15 @@ SET_PROPERTY(TARGET asm2wasm PROPERTY CXX_STANDARD 11)
 SET_PROPERTY(TARGET asm2wasm PROPERTY CXX_STANDARD_REQUIRED ON)
 INSTALL(TARGETS asm2wasm DESTINATION ${CMAKE_INSTALL_BINDIR})
 
-SET(wasm2asm_SOURCES
-  src/tools/wasm2asm.cpp
+SET(wasm2js_SOURCES
+  src/tools/wasm2js.cpp
 )
-ADD_EXECUTABLE(wasm2asm
-               ${wasm2asm_SOURCES})
-TARGET_LINK_LIBRARIES(wasm2asm passes wasm asmjs emscripten-optimizer ir cfg support)
-SET_PROPERTY(TARGET wasm2asm PROPERTY CXX_STANDARD 11)
-SET_PROPERTY(TARGET wasm2asm PROPERTY CXX_STANDARD_REQUIRED ON)
-INSTALL(TARGETS wasm2asm DESTINATION ${CMAKE_INSTALL_BINDIR})
+ADD_EXECUTABLE(wasm2js
+               ${wasm2js_SOURCES})
+TARGET_LINK_LIBRARIES(wasm2js passes wasm asmjs emscripten-optimizer ir cfg support)
+SET_PROPERTY(TARGET wasm2js PROPERTY CXX_STANDARD 11)
+SET_PROPERTY(TARGET wasm2js PROPERTY CXX_STANDARD_REQUIRED ON)
+INSTALL(TARGETS wasm2js DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 SET(wasm-emscripten-finalize_SOURCES
   src/tools/wasm-emscripten-finalize.cpp

README.md

@@ -12,7 +12,7 @@ Compilers built using Binaryen include
 
  * [`asm2wasm`](https://github.com/WebAssembly/binaryen/blob/master/src/asm2wasm.h) which compiles asm.js to WebAssembly
  * [`AssemblyScript`](https://github.com/AssemblyScript/assemblyscript) which compiles TypeScript to Binaryen IR
- * [`wasm2asm`](https://github.com/WebAssembly/binaryen/blob/master/src/wasm2asm.h) which compiles WebAssembly to asm.js
+ * [`wasm2js`](https://github.com/WebAssembly/binaryen/blob/master/src/wasm2js.h) which compiles WebAssembly to JS
  * [`Asterius`](https://github.com/tweag/asterius) which compiles Haskell to WebAssembly
 
 Binaryen also provides a set of **toolchain utilities** that can
@@ -64,7 +64,7 @@ This repository contains code that builds the following tools in `bin/`:
  * **wasm-dis**: Un-assembles WebAssembly in binary format into text format (going through Binaryen IR).
  * **wasm-opt**: Loads WebAssembly and runs Binaryen IR passes on it.
  * **asm2wasm**: An asm.js-to-WebAssembly compiler, using Emscripten's asm optimizer infrastructure. This is used by Emscripten in Binaryen mode when it uses Emscripten's fastcomp asm.js backend.
- * **wasm2asm**: A WebAssembly-to-asm.js compiler (still experimental).
+ * **wasm2js**: A WebAssembly-to-JS compiler (still experimental).
  * **wasm-merge**: Combines wasm files into a single big wasm file (without sophisticated linking).
  * **wasm-ctor-eval**: A tool that can execute C++ global constructors ahead of time. Used by Emscripten.
  * **wasm-emscripten-finalize**: Takes a wasm binary produced by llvm+lld and performs emscripten-specific passes over it.
@@ -212,11 +212,11 @@ This is separate from that. `asm2wasm` focuses on compiling asm.js to WebAssembl
 
 * How about compiling WebAssembly to asm.js (the opposite direction of `asm2wasm`)? Wouldn't that be useful for polyfilling?
 
-Experimentation with this is happening, in `wasm2asm`.
+Experimentation with this is happening, in `wasm2js`.
 
 This would be useful, but it is a much harder task, due to some decisions made in WebAssembly. For example, WebAssembly can have control flow nested inside expressions, which can't directly map to asm.js. It could be supported by outlining the code to another function, or to compiling it down into new basic blocks and control-flow-free instructions, but it is hard to do so in a way that is both fast to do and emits code that is fast to execute. On the other hand, compiling asm.js to WebAssembly is almost straightforward.
 
-We just have to do more work on `wasm2asm` and see how efficient we can make it.
+We just have to do more work on `wasm2js` and see how efficient we can make it.
 
 * Can `asm2wasm` compile any asm.js code?
 

auto_update_tests.py

@@ -22,10 +22,10 @@
 from scripts.test.support import run_command, split_wast, node_test_glue, node_has_webassembly
 from scripts.test.shared import (
     ASM2WASM, MOZJS, NODEJS, WASM_OPT, WASM_AS, WASM_DIS,
-    WASM_CTOR_EVAL, WASM_MERGE, WASM_REDUCE, WASM2ASM, WASM_METADCE,
+    WASM_CTOR_EVAL, WASM_MERGE, WASM_REDUCE, WASM2JS, WASM_METADCE,
     WASM_EMSCRIPTEN_FINALIZE, BINARYEN_INSTALL_DIR, BINARYEN_JS,
     files_with_pattern, has_shell_timeout, options)
-from scripts.test.wasm2asm import tests, spec_tests, extra_wasm2asm_tests, assert_tests, wasm2asm_dir
+from scripts.test.wasm2js import tests, spec_tests, extra_wasm2js_tests, assert_tests, wasm2js_dir, wasm2js_blacklist
 
 
 def update_asm_js_tests():
@@ -339,25 +339,28 @@ def update_ctor_eval_tests():
         o.write(actual)
 
 
-def update_wasm2asm_tests():
-  print '\n[ checking wasm2asm ]\n'
-  for wasm in tests + spec_tests + extra_wasm2asm_tests:
+def update_wasm2js_tests():
+  print '\n[ checking wasm2js ]\n'
+  for wasm in tests + spec_tests + extra_wasm2js_tests:
     if not wasm.endswith('.wast'):
       continue
 
+    if os.path.basename(wasm) in wasm2js_blacklist:
+      continue
+
     asm = os.path.basename(wasm).replace('.wast', '.2asm.js')
-    expected_file = os.path.join(wasm2asm_dir, asm)
+    expected_file = os.path.join(wasm2js_dir, asm)
 
-    # we run wasm2asm on tests and spec tests only if the output
+    # we run wasm2js on tests and spec tests only if the output
     # exists - only some work so far. the tests in extra are in
-    # the test/wasm2asm dir and so are specific to wasm2asm, and
+    # the test/wasm2js dir and so are specific to wasm2js, and
     # we run all of those.
-    if wasm not in extra_wasm2asm_tests and not os.path.exists(expected_file):
+    if wasm not in extra_wasm2js_tests and not os.path.exists(expected_file):
       continue
 
     print '..', wasm
 
-    cmd = WASM2ASM + [os.path.join('test', wasm)]
+    cmd = WASM2JS + [os.path.join('test', wasm)]
     out = run_command(cmd)
     with open(expected_file, 'w') as o:
       o.write(out)
@@ -370,7 +373,7 @@ def update_wasm2asm_tests():
     asserts_expected_file = os.path.join('test', asserts)
     traps_expected_file = os.path.join('test', traps)
 
-    cmd = WASM2ASM + [os.path.join(wasm2asm_dir, wasm), '--allow-asserts']
+    cmd = WASM2JS + [os.path.join(wasm2js_dir, wasm), '--allow-asserts']
     out = run_command(cmd)
     with open(asserts_expected_file, 'w') as o:
       o.write(out)
@@ -423,7 +426,7 @@ def main():
   update_wasm_merge_tests()
   update_binaryen_js_tests()
   update_ctor_eval_tests()
-  update_wasm2asm_tests()
+  update_wasm2js_tests()
   update_metadce_tests()
   update_reduce_tests()
 

check.py

@@ -31,7 +31,7 @@
 
 import scripts.test.asm2wasm as asm2wasm
 import scripts.test.lld as lld
-import scripts.test.wasm2asm as wasm2asm
+import scripts.test.wasm2js as wasm2js
 
 if options.interpreter:
   print '[ using wasm interpreter at "%s" ]' % options.interpreter
@@ -648,7 +648,7 @@ def main():
   run_spec_tests()
   run_binaryen_js_tests()
   lld.test_wasm_emscripten_finalize()
-  wasm2asm.test_wasm2asm()
+  wasm2js.test_wasm2js()
   run_validator_tests()
   if has_vanilla_emcc and has_vanilla_llvm and 0:
     run_vanilla_tests()

scripts/test/node-esm-loader.mjs

@@ -0,0 +1,32 @@
+// originally lifted from https://nodejs.org/api/esm.html
+
+import path from 'path';
+import process from 'process';
+import Module from 'module';
+
+const builtins = Module.builtinModules;
+
+const baseURL = new URL('file://');
+baseURL.pathname = `${process.cwd()}/`;
+
+export function resolve(specifier, parentModuleURL = baseURL, defaultResolve) {
+  if (builtins.includes(specifier)) {
+    return {
+      url: specifier,
+      format: 'builtin'
+    };
+  }
+  // Resolve the 'spectest' module to our special module which has some builtins
+  if (specifier == 'spectest') {
+    const resolved = new URL('./scripts/test/spectest.js', parentModuleURL);
+    return {
+      url: resolved.href,
+      format: 'esm'
+    };
+  }
+  const resolved = new URL(specifier, parentModuleURL);
+  return {
+    url: resolved.href,
+    format: 'esm'
+  };
+}

scripts/test/shared.py

@@ -166,7 +166,7 @@ def is_exe(fpath):
 WASM_AS = [os.path.join(options.binaryen_bin, 'wasm-as')]
 WASM_DIS = [os.path.join(options.binaryen_bin, 'wasm-dis')]
 ASM2WASM = [os.path.join(options.binaryen_bin, 'asm2wasm')]
-WASM2ASM = [os.path.join(options.binaryen_bin, 'wasm2asm')]
+WASM2JS = [os.path.join(options.binaryen_bin, 'wasm2js')]
 WASM_CTOR_EVAL = [os.path.join(options.binaryen_bin, 'wasm-ctor-eval')]
 WASM_SHELL = [os.path.join(options.binaryen_bin, 'wasm-shell')]
 WASM_MERGE = [os.path.join(options.binaryen_bin, 'wasm-merge')]

scripts/test/spectest.js

@@ -0,0 +1,3 @@
+export function print(...args) {
+  console.log(...args);
+}

scripts/test/support.py

@@ -147,7 +147,7 @@ def to_end(j):
 
 
 def run_command(cmd, expected_status=0, stderr=None,
-                expected_err=None, err_contains=False):
+                expected_err=None, err_contains=False, err_ignore=None):
   if expected_err is not None:
     assert stderr == subprocess.PIPE or stderr is None,\
         "Can't redirect stderr if using expected_err"
@@ -158,11 +158,13 @@ def run_command(cmd, expected_status=0, stderr=None,
   code = proc.returncode
   if expected_status is not None and code != expected_status:
     raise Exception(('run_command failed (%s)' % code, out + str(err or '')))
-  err_correct = expected_err is None or \
-      (expected_err in err if err_contains else expected_err == err)
-  if not err_correct:
-    raise Exception(('run_command unexpected stderr',
-                     "expected '%s', actual '%s'" % (expected_err, err)))
+  if expected_err is not None:
+    if err_ignore is not None:
+      err = "\n".join([line for line in err.split('\n') if err_ignore not in line])
+    err_correct = expected_err in err if err_contains else expected_err == err
+    if not err_correct:
+      raise Exception(('run_command unexpected stderr',
+                       "expected '%s', actual '%s'" % (expected_err, err)))
   return out
 
 

scripts/test/wasm2asm.py scripts/test/wasm2js.pyscripts/test/wasm2asm.py renamed to scripts/test/wasm2js.py

@@ -18,7 +18,7 @@
 
 from support import run_command
 from shared import (
-    WASM2ASM, MOZJS, NODEJS, fail_if_not_identical, options, tests,
+    WASM2JS, MOZJS, NODEJS, fail_if_not_identical, options, tests,
     fail_if_not_identical_to_file
 )
 
@@ -27,59 +27,59 @@
 spec_tests = [os.path.join(spec_dir, t)
               for t in sorted(os.listdir(spec_dir))
               if '.fail' not in t]
-wasm2asm_dir = os.path.join(options.binaryen_test, 'wasm2asm')
-extra_wasm2asm_tests = [os.path.join(wasm2asm_dir, t) for t in
-                        sorted(os.listdir(wasm2asm_dir))]
-assert_tests = ['wasm2asm.wast.asserts']
+wasm2js_dir = os.path.join(options.binaryen_test, 'wasm2js')
+extra_wasm2js_tests = [os.path.join(wasm2js_dir, t) for t in
+                       sorted(os.listdir(wasm2js_dir))]
+assert_tests = ['wasm2js.wast.asserts']
+# These tests exercise functionality not supported by wasm2js
+wasm2js_blacklist = ['empty_imported_table.wast']
 
 
-def test_wasm2asm_output():
-  for wasm in tests + spec_tests + extra_wasm2asm_tests:
+def test_wasm2js_output():
+  for wasm in tests + spec_tests + extra_wasm2js_tests:
     if not wasm.endswith('.wast'):
       continue
+    basename = os.path.basename(wasm)
+    if basename in wasm2js_blacklist:
+      continue
 
-    asm = os.path.basename(wasm).replace('.wast', '.2asm.js')
-    expected_file = os.path.join(wasm2asm_dir, asm)
+    asm = basename.replace('.wast', '.2asm.js')
+    expected_file = os.path.join(wasm2js_dir, asm)
 
     if not os.path.exists(expected_file):
       continue
 
     print '..', wasm
 
-    cmd = WASM2ASM + [os.path.join(options.binaryen_test, wasm)]
+    cmd = WASM2JS + [os.path.join(options.binaryen_test, wasm)]
     out = run_command(cmd)
     fail_if_not_identical_to_file(out, expected_file)
 
     if not NODEJS and not MOZJS:
       print 'No JS interpreters. Skipping spec tests.'
       continue
 
-    open('a.2asm.js', 'w').write(out)
+    open('a.2asm.mjs', 'w').write(out)
 
     cmd += ['--allow-asserts']
     out = run_command(cmd)
 
-    open('a.2asm.asserts.js', 'w').write(out)
+    open('a.2asm.asserts.mjs', 'w').write(out)
 
-    # verify asm.js is valid js
+    # verify asm.js is valid js, note that we're using --experimental-modules
+    # to enable ESM syntax and we're also passing a custom loader to handle the
+    # `spectest` module in our tests.
     if NODEJS:
-      out = run_command([NODEJS, 'a.2asm.js'])
+      node = [NODEJS, '--experimental-modules', '--loader', './scripts/test/node-esm-loader.mjs']
+      cmd = node[:]
+      cmd.append('a.2asm.mjs')
+      out = run_command(cmd)
       fail_if_not_identical(out, '')
-      out = run_command([NODEJS, 'a.2asm.asserts.js'], expected_err='')
+      cmd = node[:]
+      cmd.append('a.2asm.asserts.mjs')
+      out = run_command(cmd, expected_err='', err_ignore='The ESM module loader is experimental')
       fail_if_not_identical(out, '')
 
-    if MOZJS:
-      # verify asm.js validates, if this is asm.js code (we emit
-      # almost-asm instead when we need to)
-      if 'use asm' in open('a.2asm.js').read():
-        # check only subset of err because mozjs emits timing info
-        out = run_command([MOZJS, '-w', 'a.2asm.js'],
-                          expected_err='Successfully compiled asm.js code',
-                          err_contains=True)
-        fail_if_not_identical(out, '')
-        out = run_command([MOZJS, 'a.2asm.asserts.js'], expected_err='')
-        fail_if_not_identical(out, '')
-
 
 def test_asserts_output():
   for wasm in assert_tests:
@@ -90,8 +90,8 @@ def test_asserts_output():
     asserts_expected_file = os.path.join(options.binaryen_test, asserts)
     traps_expected_file = os.path.join(options.binaryen_test, traps)
 
-    wasm = os.path.join(wasm2asm_dir, wasm)
-    cmd = WASM2ASM + [wasm, '--allow-asserts']
+    wasm = os.path.join(wasm2js_dir, wasm)
+    cmd = WASM2JS + [wasm, '--allow-asserts']
     out = run_command(cmd)
     fail_if_not_identical_to_file(out, asserts_expected_file)
 
@@ -100,11 +100,11 @@ def test_asserts_output():
     fail_if_not_identical_to_file(out, traps_expected_file)
 
 
-def test_wasm2asm():
-  print '\n[ checking wasm2asm testcases... ]\n'
-  test_wasm2asm_output()
+def test_wasm2js():
+  print '\n[ checking wasm2js testcases... ]\n'
+  test_wasm2js_output()
   test_asserts_output()
 
 
 if __name__ == "__main__":
-  test_wasm2asm()
+  test_wasm2js()

src/binaryen-c.cpp

@@ -29,7 +29,7 @@
 #include "wasm-printing.h"
 #include "wasm-s-parser.h"
 #include "wasm-validator.h"
-#include "wasm2asm.h"
+#include "wasm2js.h"
 #include "cfg/Relooper.h"
 #include "ir/utils.h"
 #include "shell-interface.h"
@@ -2008,9 +2008,9 @@ void BinaryenModulePrintAsmjs(BinaryenModuleRef module) {
   }
 
   Module* wasm = (Module*)module;
-  Wasm2AsmBuilder::Flags builderFlags;
-  Wasm2AsmBuilder wasm2asm(builderFlags);
-  Ref asmjs = wasm2asm.processWasm(wasm);
+  Wasm2JSBuilder::Flags builderFlags;
+  Wasm2JSBuilder wasm2js(builderFlags);
+  Ref asmjs = wasm2js.processWasm(wasm);
   JSPrinter jser(true, true, asmjs);
   jser.printAst();