Support ES6 module syntax in Terser and Closure

Let's see if this passes the CI.

(this commit should probably be split into its own PR)

Author: kleisauke

emcc.py

@@ -3094,10 +3094,9 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
   generated_text_files_with_native_eols = []
 
   if settings.MODULARIZE:
-    modularize()
-
-  if settings.USE_CLOSURE_COMPILER:
-    module_export_name_substitution()
+    final_js = post_modularize(final_js)
+  elif settings.USE_CLOSURE_COMPILER:
+    final_js = module_export_name_substitution(final_js)
 
   # Run a final optimization pass to clean up items that were not possible to
   # optimize by Closure, or unoptimalities that were left behind by processing
@@ -3109,18 +3108,6 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
     # mode)
     final_js = building.closure_compiler(final_js, pretty=False, advanced=False, extra_closure_args=options.closure_args)
 
-  # Unmangle previously mangled `import.meta` and `await import` references in
-  # both main code and libraries.
-  # See also: `preprocess` in parseTools.js.
-  if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
-    src = read_file(final_js)
-    final_js += '.esmeta.js'
-    write_file(final_js, src
-               .replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
-               .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))
-    shared.get_temp_files().note(final_js)
-    save_intermediate('es6-module')
-
   # Apply pre and postjs files
   if options.extern_pre_js or options.extern_post_js:
     logger.debug('applying extern pre/postjses')
@@ -3566,6 +3553,11 @@ def phase_binaryen(target, options, wasm_target):
   # after generating the wasm, do some final operations
 
   if final_js:
+    # Note: this needs to be run before the other acorn-optimizer.js passes,
+    # to safely use the 'await' keyword when doing a multi-environment build.
+    if settings.MODULARIZE:
+      final_js = pre_modularize(final_js)
+
     if settings.SUPPORT_BIG_ENDIAN:
       final_js = building.little_endian_heap(final_js)
 
@@ -3706,10 +3698,9 @@ def node_es6_imports():
 '''
 
 
-def modularize():
-  global final_js
-  logger.debug(f'Modularizing, assigning to var {settings.EXPORT_NAME}')
-  src = read_file(final_js)
+def pre_modularize(js_file):
+  logger.debug(f'Pre-modularizing, assigning to var {settings.EXPORT_NAME}')
+  src = read_file(js_file)
 
   # Multi-environment ES6 builds require an async function
   async_emit = ''
@@ -3718,16 +3709,14 @@ def modularize():
      shared.target_environment_may_be('web'):
     async_emit = 'async '
 
-  return_value = settings.EXPORT_NAME
+  return_value = 'Module'
   if settings.WASM_ASYNC_COMPILATION:
     return_value += '.ready'
   if not settings.EXPORT_READY_PROMISE:
     return_value = '{}'
 
   src = '''
 %(maybe_async)sfunction(%(EXPORT_NAME)s) {
-  %(EXPORT_NAME)s = %(EXPORT_NAME)s || {};
-
 %(src)s
 
   return %(return_value)s
@@ -3755,21 +3744,53 @@ def modularize():
       script_url = "typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined"
       if shared.target_environment_may_be('node'):
         script_url_node = "if (typeof __filename !== 'undefined') _scriptDir = _scriptDir || __filename;"
-    src = '''%(node_imports)s
+    src = '''
 var %(EXPORT_NAME)s = (() => {
   var _scriptDir = %(script_url)s;
   %(script_url_node)s
   return (%(src)s);
 })();
 ''' % {
-      'node_imports': node_es6_imports(),
       'EXPORT_NAME': settings.EXPORT_NAME,
       'script_url': script_url,
       'script_url_node': script_url_node,
       'src': src
     }
 
-  final_js += '.modular.js'
+  final_js = js_file + '.pre-modular.js'
+  with open(final_js, 'w', encoding='utf-8') as f:
+    f.write(src)
+
+    # Ensure Closure compiler is aware of that
+    # settings.EXPORT_NAME needs to be exported.
+    # https://stackoverflow.com/questions/46092308
+    if settings.USE_CLOSURE_COMPILER:
+      f.write('window["%(EXPORT_NAME)s"] = %(EXPORT_NAME)s;' % {
+          'EXPORT_NAME': settings.EXPORT_NAME
+      })
+
+  shared.get_temp_files().note(final_js)
+  save_intermediate('pre-modularized')
+  return final_js
+
+
+def post_modularize(js_file):
+  src = read_file(js_file)
+
+  # Replace variants of `window.Module` with `var Module`
+  # and remove the line containing `export{};`
+  # See also: `pre_modularize`
+  if settings.USE_CLOSURE_COMPILER:
+    src = src.replace('window.%s' % settings.EXPORT_NAME,
+                      'var %s' % settings.EXPORT_NAME)
+    src = src.replace('window["%s"]' % settings.EXPORT_NAME,
+                      'var %s' % settings.EXPORT_NAME)
+    src = src.replace('export{};\n', '')
+
+  # Prepend static import declarations, if any
+  src = node_es6_imports() + src
+
+  final_js = js_file + '.post-modular.js'
   with open(final_js, 'w', encoding='utf-8') as f:
     f.write(src)
 
@@ -3788,14 +3809,14 @@ def modularize():
 ''' % {'EXPORT_NAME': settings.EXPORT_NAME})
 
   shared.get_temp_files().note(final_js)
-  save_intermediate('modularized')
+  save_intermediate('post-modularized')
+  return final_js
 
 
-def module_export_name_substitution():
-  global final_js
+def module_export_name_substitution(js_file):
   logger.debug(f'Private module export name substitution with {settings.EXPORT_NAME}')
-  src = read_file(final_js)
-  final_js += '.module_export_name_substitution.js'
+  src = read_file(js_file)
+  final_js = js_file + '.module_export_name_substitution.js'
   if settings.MINIMAL_RUNTIME and not settings.ENVIRONMENT_MAY_BE_NODE and not settings.ENVIRONMENT_MAY_BE_SHELL:
     # On the web, with MINIMAL_RUNTIME, the Module object is always provided
     # via the shell html in order to provide the .asm.js/.wasm content.
@@ -3804,8 +3825,10 @@ def module_export_name_substitution():
     replacement = "typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {}" % {"EXPORT_NAME": settings.EXPORT_NAME}
   src = re.sub(r'{\s*[\'"]?__EMSCRIPTEN_PRIVATE_MODULE_EXPORT_NAME_SUBSTITUTION__[\'"]?:\s*1\s*}', replacement, src)
   write_file(final_js, src)
+
   shared.get_temp_files().note(final_js)
   save_intermediate('module_export_name_substitution')
+  return final_js
 
 
 def generate_traditional_runtime_html(target, options, js_target, target_basename,

emscripten.py

@@ -203,6 +203,12 @@ def compile_settings():
                              cwd=path_from_root('src'), env=env, encoding='utf-8')
   assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?'
   glue, forwarded_data = out.split('//FORWARDED_DATA:')
+
+  # Unmangle previously mangled `import.meta` references in library_*.js.
+  # See also: `LibraryManager.load` in modules.js.
+  if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
+    glue = glue.replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
+
   return glue, forwarded_data
 
 

src/closure-externs/closure-externs.js

@@ -11,11 +11,7 @@
  * The closure_compiler() method in tools/shared.py refers to this file when calling closure.
  */
 
-// Special placeholder for `import.meta` and `await import`.
-var EMSCRIPTEN$IMPORT$META;
-var EMSCRIPTEN$AWAIT$IMPORT;
-
-// Don't minify createRequire
+// Don't minify createRequire (i.e. when doing -sEXPORT_ES6 + -sENVIRONMENT=node)
 var createRequire;
 
 // Closure externs used by library_sockfs.js
@@ -213,14 +209,6 @@ var close;
  */
 var ENVIRONMENT_IS_FETCH_WORKER;
 
-// Due to the way MODULARIZE works, Closure is run on generated code that does not define _scriptDir,
-// but only after MODULARIZE has finished, _scriptDir is injected to the generated code.
-// Therefore it cannot be minified.
-/**
- * @suppress {duplicate, undefinedVars}
- */
-var _scriptDir;
-
 // Closure run on asm.js uses a hack to execute only on shell code, declare externs needed for it.
 /**
  * @suppress {undefinedVars}

src/modules.js

@@ -212,6 +212,13 @@ global.LibraryManager = {
       }
       try {
         processed = processMacros(preprocess(src, filename));
+        if (EXPORT_ES6 && USE_ES6_IMPORT_META) {
+          // `eval` doesn't support module syntax; to allow it, we need to temporarily
+          // replace `import.meta` usages with placeholders during the JS compile phase,
+          // then in Python we reverse this replacement.
+          // See also: `compile_settings` in emscripten.py.
+          processed = processed.replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META')
+        }
         eval(processed);
       } catch (e) {
         const details = [e, e.lineNumber ? `line number: ${e.lineNumber}` : ''];

src/parseTools.js

@@ -35,16 +35,6 @@ function processMacros(text) {
 // Param filenameHint can be passed as a description to identify the file that is being processed, used
 // to locate errors for reporting and for html files to stop expansion between <style> and </style>.
 function preprocess(text, filenameHint) {
-  if (EXPORT_ES6 && USE_ES6_IMPORT_META) {
-    // `eval`, Terser and Closure don't support module syntax; to allow it,
-    // we need to temporarily replace `import.meta` and `await import` usages
-    // with placeholders during preprocess phase, and back after all the other ops.
-    // See also: `phase_final_emitting` in emcc.py.
-    text = text
-      .replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META')
-      .replace(/\bawait import\b/g, 'EMSCRIPTEN$AWAIT$IMPORT');
-  }
-
   const IGNORE = 0;
   const SHOW = 1;
   // This state is entered after we have shown one of the block of an if/elif/else sequence.

src/shell.js

@@ -21,7 +21,7 @@
 // after the generated code, you will need to define   var Module = {};
 // before the code. Then that object will be used in the code, and you
 // can continue to use Module afterwards as well.
-#if USE_CLOSURE_COMPILER
+#if USE_CLOSURE_COMPILER && !MODULARIZE
 // if (!Module)` is crucial for Closure Compiler here as it will otherwise replace every `Module` occurrence with a string
 var /** @type {{
   noImageDecoding: boolean,
@@ -36,8 +36,12 @@ var /** @type {{
  */ Module;
 if (!Module) /** @suppress{checkTypes}*/Module = {"__EMSCRIPTEN_PRIVATE_MODULE_EXPORT_NAME_SUBSTITUTION__":1};
 #else
-var Module = typeof {{{ EXPORT_NAME }}} != 'undefined' ? {{{ EXPORT_NAME }}} : {};
-#endif // USE_CLOSURE_COMPILER
+// Under `MODULARIZE`, avoid re-declaring the Module variable it's identical to EXPORT_NAME
+#if !MODULARIZE || EXPORT_NAME != 'Module'
+var
+#endif
+  Module = typeof {{{ EXPORT_NAME }}} != 'undefined' ? {{{ EXPORT_NAME }}} : {};
+#endif // USE_CLOSURE_COMPILER && !MODULARIZE
 
 #if POLYFILL
 #if ((MAYBE_WASM2JS && WASM != 2) || MODULARIZE) && (MIN_CHROME_VERSION < 33 || MIN_EDGE_VERSION < 12 || MIN_FIREFOX_VERSION < 29 || MIN_IE_VERSION != TARGET_NOT_SUPPORTED || MIN_SAFARI_VERSION < 80000) // https://caniuse.com/#feat=promises
@@ -195,7 +199,6 @@ if (ENVIRONMENT_IS_NODE) {
   // TODO: Swap all `require()`'s with `import()`'s?
 #if EXPORT_ES6 && ENVIRONMENT_MAY_BE_WEB
   const { createRequire } = await import('module');
-  /** @suppress{duplicate} */
   var require = createRequire(import.meta.url);
 #endif
   // These modules will usually be used on Node.js. Load them eagerly to avoid

src/shell_minimal.js

@@ -4,18 +4,25 @@
  * SPDX-License-Identifier: MIT
  */
 
-#if USE_CLOSURE_COMPILER
+#if USE_CLOSURE_COMPILER && !MODULARIZE
 // if (!Module)` is crucial for Closure Compiler here as it will
 // otherwise replace every `Module` occurrence with the object below
 var /** @type{Object} */ Module;
 if (!Module) /** @suppress{checkTypes}*/Module = {"__EMSCRIPTEN_PRIVATE_MODULE_EXPORT_NAME_SUBSTITUTION__":1};
-#elif ENVIRONMENT_MAY_BE_NODE || ENVIRONMENT_MAY_BE_SHELL
+#else
+// Under `MODULARIZE`, avoid re-declaring the Module variable it's identical to EXPORT_NAME
+#if !MODULARIZE || EXPORT_NAME != 'Module'
+var
+#endif
 // When running on the web we expect Module to be defined externally, in the
 // HTML.  Otherwise we must define it here before its first use
-var Module = typeof {{{ EXPORT_NAME }}} != 'undefined' ? {{{ EXPORT_NAME }}} : {};
+  Module =
+#if ENVIRONMENT_MAY_BE_NODE || ENVIRONMENT_MAY_BE_SHELL
+    typeof {{{ EXPORT_NAME }}} != 'undefined' ? {{{ EXPORT_NAME }}} : {};
 #else
-var Module = {{{ EXPORT_NAME }}};
-#endif // USE_CLOSURE_COMPILER
+    {{{ EXPORT_NAME }}};
+#endif // ENVIRONMENT_MAY_BE_NODE || ENVIRONMENT_MAY_BE_SHELL
+#endif // USE_CLOSURE_COMPILER && !MODULARIZE
 
 #if MODULARIZE && EXPORT_READY_PROMISE
 // Set up the promise that indicates the Module is initialized

test/export_module.js

@@ -1,5 +1,2 @@
-function doNothing() {
-  return false;
-}
-
-export {doNothing};
+const doNothing = () => false;
+export default doNothing;

test/test_other.py

@@ -241,14 +241,31 @@ def test_emcc_generate_config(self):
   @parameterized({
     '': ([],),
     'node': (['-sENVIRONMENT=node'],),
+    'optimize': (['-O3'],),  # i.e., OPT_LEVEL >= 2, minified with tools/acorn-optimizer.js
   })
   def test_emcc_output_mjs(self, args):
+    optimized = '-O3' in args
     create_file('extern-post.js', 'await Module();')
     self.run_process([EMCC, '-o', 'hello_world.mjs',
                       '--extern-post-js', 'extern-post.js',
                       test_file('hello_world.c')] + args)
     src = read_file('hello_world.mjs')
     self.assertContained('export default Module;', src)
+    self.assertContainedIf("new URL('hello_world.wasm', import.meta.url)", src, not optimized)
+    self.assertContainedIf('new URL("hello_world.wasm",import.meta.url)', src, optimized)
+    self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
+
+  @parameterized({
+    '': ([],),
+    'node': (['-sENVIRONMENT=node'],),
+  })
+  def test_emcc_output_mjs_closure(self, args):
+    create_file('extern-post.js', 'await Module();')
+    self.run_process([EMCC, '-o', 'hello_world.mjs',
+                      '--extern-post-js', 'extern-post.js',
+                      test_file('hello_world.c'), '--closure=1'] + args)
+    src = read_file('hello_world.mjs')
+    self.assertContained('new URL("hello_world.wasm", import.meta.url)', src)
     self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
 
   @parameterized({
@@ -282,15 +299,6 @@ def test_emcc_output_worker_mjs_single_file(self):
     self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url))", src)
     self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
 
-  def test_emcc_output_mjs_closure(self):
-    create_file('extern-post.js', 'await Module();')
-    self.run_process([EMCC, '-o', 'hello_world.mjs',
-                      '--extern-post-js', 'extern-post.js',
-                      test_file('hello_world.c'), '--closure=1'])
-    src = read_file('hello_world.mjs')
-    self.assertContained('new URL("hello_world.wasm", import.meta.url)', src)
-    self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
-
   def test_emcc_output_mjs_web_no_import_meta(self):
     # Ensure we don't emit import.meta.url at all for:
     # ENVIRONMENT=web + EXPORT_ES6 + USE_ES6_IMPORT_META=0
@@ -316,9 +324,20 @@ def test_export_es6_node_requires_import_meta(self):
     self.assertContained('EXPORT_ES6 and ENVIRONMENT=*node* requires USE_ES6_IMPORT_META to be set', err)
 
   def test_export_es6_allows_export_in_post_js(self):
-    self.run_process([EMCC, test_file('hello_world.c'), '-O3', '-sEXPORT_ES6', '--post-js', test_file('export_module.js')])
+    self.run_process([EMCC, test_file('hello_world.c'), '-O3', '-sEXPORT_ES6',
+                      '--post-js', test_file('export_module.js')])
     src = read_file('a.out.js')
-    self.assertContained('export{doNothing};', src)
+    self.assertContained('export default doNothing;', src)
+
+  # FIXME: This only works for multi-environment builds.
+  # Should we always emit an async function for EXPORT_ES6?
+  def test_export_es6_allows_await_in_pre_js(self):
+    create_file('pre.js', 'await console.log("Wasm rocks!");')
+    self.run_process([EMCC, '-o', 'hello_world.mjs', '-O3',
+                      test_file('hello_world.c'),
+                      '--pre-js', 'pre.js'])
+    src = read_file('hello_world.mjs')
+    self.assertContained('await console.log("Wasm rocks!");', src)
 
   def test_emcc_out_file(self):
     # Verify that "-ofile" works in addition to "-o" "file"