diff --git a/ChangeLog.md b/ChangeLog.md index a8c59c499f9d0..e5b6b378efd29 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -31,6 +31,7 @@ See docs/process.md for more on how version tagging works. overflow will trap rather corrupting global data first). This should not be a user-visible change (unless your program does something very odd such depending on the specific location of stack data in memory). (#18154) +- Add support for `-sEXPORT_ES6`/`*.mjs` on Node.js. (#17915) 3.1.25 - 11/08/22 ----------------- diff --git a/emcc.py b/emcc.py index 19d0c939814b0..b240ecc527908 100755 --- a/emcc.py +++ b/emcc.py @@ -2338,11 +2338,17 @@ def check_memory_setting(setting): if 'MAXIMUM_MEMORY' in user_settings and not settings.ALLOW_MEMORY_GROWTH: diagnostics.warning('unused-command-line-argument', 'MAXIMUM_MEMORY is only meaningful with ALLOW_MEMORY_GROWTH') - if settings.EXPORT_ES6 and not settings.MODULARIZE: - # EXPORT_ES6 requires output to be a module - if 'MODULARIZE' in user_settings: - exit_with_error('EXPORT_ES6 requires MODULARIZE to be set') - settings.MODULARIZE = 1 + if settings.EXPORT_ES6: + if not settings.MODULARIZE: + # EXPORT_ES6 requires output to be a module + if 'MODULARIZE' in user_settings: + exit_with_error('EXPORT_ES6 requires MODULARIZE to be set') + settings.MODULARIZE = 1 + if shared.target_environment_may_be('node') and not settings.USE_ES6_IMPORT_META: + # EXPORT_ES6 + ENVIRONMENT=*node* requires the use of import.meta.url + if 'USE_ES6_IMPORT_META' in user_settings: + exit_with_error('EXPORT_ES6 and ENVIRONMENT=*node* requires USE_ES6_IMPORT_META to be set') + settings.USE_ES6_IMPORT_META = 1 if settings.MODULARIZE and not settings.DECLARE_ASM_MODULE_EXPORTS: # When MODULARIZE option is used, currently requires declaring all module exports @@ -3103,13 +3109,17 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): # mode) final_js = building.closure_compiler(final_js, pretty=False, advanced=False, extra_closure_args=options.closure_args) - # Unmangle previously mangled `import.meta` references in both main code and libraries. + # Unmangle previously mangled `import.meta` and `await import` references in + # both main code and libraries. # See also: `preprocess` in parseTools.js. if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META: src = read_file(final_js) final_js += '.esmeta.js' - write_file(final_js, src.replace('EMSCRIPTEN$IMPORT$META', 'import.meta')) - save_intermediate('es6-import-meta') + write_file(final_js, src + .replace('EMSCRIPTEN$IMPORT$META', 'import.meta') + .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import')) + shared.get_temp_files().note(final_js) + save_intermediate('es6-module') # Apply pre and postjs files if options.extern_pre_js or options.extern_post_js: @@ -3681,11 +3691,33 @@ def preprocess_wasm2js_script(): write_file(final_js, js) +def node_es6_imports(): + if not settings.EXPORT_ES6 or not shared.target_environment_may_be('node'): + return '' + + # Multi-environment builds uses `await import` in `shell.js` + if shared.target_environment_may_be('web'): + return '' + + # Use static import declaration if we only target Node.js + return ''' +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); +''' + + def modularize(): global final_js logger.debug(f'Modularizing, assigning to var {settings.EXPORT_NAME}') src = read_file(final_js) + # Multi-environment ES6 builds require an async function + async_emit = '' + if settings.EXPORT_ES6 and \ + shared.target_environment_may_be('node') and \ + shared.target_environment_may_be('web'): + async_emit = 'async ' + return_value = settings.EXPORT_NAME if settings.WASM_ASYNC_COMPILATION: return_value += '.ready' @@ -3693,7 +3725,7 @@ def modularize(): return_value = '{}' src = ''' -function(%(EXPORT_NAME)s) { +%(maybe_async)sfunction(%(EXPORT_NAME)s) { %(EXPORT_NAME)s = %(EXPORT_NAME)s || {}; %(src)s @@ -3701,6 +3733,7 @@ def modularize(): return %(return_value)s } ''' % { + 'maybe_async': async_emit, 'EXPORT_NAME': settings.EXPORT_NAME, 'src': src, 'return_value': return_value @@ -3711,24 +3744,25 @@ def modularize(): # document.currentScript, so a simple export declaration is enough. src = 'var %s=%s' % (settings.EXPORT_NAME, src) else: - script_url_node = "" + script_url_node = '' # When MODULARIZE this JS may be executed later, # after document.currentScript is gone, so we save it. # In EXPORT_ES6 + USE_PTHREADS the 'thread' is actually an ES6 module webworker running in strict mode, # so doesn't have access to 'document'. In this case use 'import.meta' instead. if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META: - script_url = "import.meta.url" + script_url = 'import.meta.url' else: script_url = "typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined" if shared.target_environment_may_be('node'): script_url_node = "if (typeof __filename !== 'undefined') _scriptDir = _scriptDir || __filename;" - src = ''' + src = '''%(node_imports)s var %(EXPORT_NAME)s = (() => { var _scriptDir = %(script_url)s; %(script_url_node)s return (%(src)s); })(); ''' % { + 'node_imports': node_es6_imports(), 'EXPORT_NAME': settings.EXPORT_NAME, 'script_url': script_url, 'script_url_node': script_url_node, diff --git a/src/closure-externs/closure-externs.js b/src/closure-externs/closure-externs.js index 11b08204f137f..23f4bc3eb9eeb 100644 --- a/src/closure-externs/closure-externs.js +++ b/src/closure-externs/closure-externs.js @@ -11,8 +11,12 @@ * The closure_compiler() method in tools/shared.py refers to this file when calling closure. */ -// Special placeholder for `import.meta`. +// Special placeholder for `import.meta` and `await import`. var EMSCRIPTEN$IMPORT$META; +var EMSCRIPTEN$AWAIT$IMPORT; + +// Don't minify createRequire +var createRequire; // Closure externs used by library_sockfs.js diff --git a/src/node_shell_read.js b/src/node_shell_read.js index 2a7fce592b9aa..1069a4496db0a 100644 --- a/src/node_shell_read.js +++ b/src/node_shell_read.js @@ -4,21 +4,6 @@ * SPDX-License-Identifier: MIT */ -// These modules will usually be used on Node.js. Load them eagerly to avoid -// the complexity of lazy-loading. However, for now we must guard on require() -// actually existing: if the JS is put in a .mjs file (ES6 module) and run on -// node, then we'll detect node as the environment and get here, but require() -// does not exist (since ES6 modules should use |import|). If the code actually -// uses the node filesystem then it will crash, of course, but in the case of -// code that never uses it we don't want to crash here, so the guarding if lets -// such code work properly. See discussion in -// https://github.com/emscripten-core/emscripten/pull/17851 -var fs, nodePath; -if (typeof require === 'function') { - fs = require('fs'); - nodePath = require('path'); -} - read_ = (filename, binary) => { #if SUPPORT_BASE64_EMBEDDING var ret = tryParseAsDataURI(filename); @@ -26,7 +11,9 @@ read_ = (filename, binary) => { return binary ? ret : ret.toString(); } #endif - filename = nodePath['normalize'](filename); + // We need to re-wrap `file://` strings to URLs. Normalizing isn't + // necessary in that case, the path should already be absolute. + filename = isFileURI(filename) ? new URL(filename) : nodePath.normalize(filename); return fs.readFileSync(filename, binary ? undefined : 'utf8'); }; @@ -48,7 +35,8 @@ readAsync = (filename, onload, onerror) => { onload(ret); } #endif - filename = nodePath['normalize'](filename); + // See the comment in the `read_` function. + filename = isFileURI(filename) ? new URL(filename) : nodePath.normalize(filename); fs.readFile(filename, function(err, data) { if (err) onerror(err); else onload(data.buffer); diff --git a/src/parseTools.js b/src/parseTools.js index ff84b3d964a3a..26de0b8918911 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -37,10 +37,12 @@ function processMacros(text) { function preprocess(text, filenameHint) { if (EXPORT_ES6 && USE_ES6_IMPORT_META) { // `eval`, Terser and Closure don't support module syntax; to allow it, - // we need to temporarily replace `import.meta` usages with placeholders - // during preprocess phase, and back after all the other ops. + // we need to temporarily replace `import.meta` and `await import` usages + // with placeholders during preprocess phase, and back after all the other ops. // See also: `phase_final_emitting` in emcc.py. - text = text.replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META'); + text = text + .replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META') + .replace(/\bawait import\b/g, 'EMSCRIPTEN$AWAIT$IMPORT'); } const IGNORE = 0; diff --git a/src/preamble.js b/src/preamble.js index 320652ad9a117..5519c7430be2f 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -641,7 +641,7 @@ if (Module['locateFile']) { #if EXPORT_ES6 && USE_ES6_IMPORT_META && !SINGLE_FILE // in single-file mode, repeating WASM_BINARY_FILE would emit the contents again } else { // Use bundler-friendly `new URL(..., import.meta.url)` pattern; works in browsers too. - wasmBinaryFile = new URL('{{{ WASM_BINARY_FILE }}}', import.meta.url).toString(); + wasmBinaryFile = new URL('{{{ WASM_BINARY_FILE }}}', import.meta.url).href; } #endif diff --git a/src/settings.js b/src/settings.js index 671ad3c664eb3..1659eeeece7db 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1236,7 +1236,8 @@ var EXPORT_ES6 = false; // Use the ES6 Module relative import feature 'import.meta.url' // to auto-detect WASM Module path. -// It might not be supported on old browsers / toolchains +// It might not be supported on old browsers / toolchains. This setting +// may not be disabled when Node.js is targeted (-sENVIRONMENT=*node*). // [link] var USE_ES6_IMPORT_META = true; diff --git a/src/shell.js b/src/shell.js index e704a829e1914..851022e3c5005 100644 --- a/src/shell.js +++ b/src/shell.js @@ -133,9 +133,6 @@ var ENVIRONMENT_IS_WASM_WORKER = Module['$ww']; #if SHARED_MEMORY && !MODULARIZE // In MODULARIZE mode _scriptDir needs to be captured already at the very top of the page immediately when the page is parsed, so it is generated there // before the page load. In non-MODULARIZE modes generate it here. -#if EXPORT_ES6 -var _scriptDir = import.meta.url; -#else var _scriptDir = (typeof document != 'undefined' && document.currentScript) ? document.currentScript.src : undefined; if (ENVIRONMENT_IS_WORKER) { @@ -146,8 +143,7 @@ else if (ENVIRONMENT_IS_NODE) { _scriptDir = __filename; } #endif // ENVIRONMENT_MAY_BE_NODE -#endif -#endif +#endif // SHARED_MEMORY && !MODULARIZE // `/` should be present at the end if `scriptDirectory` is not empty var scriptDirectory = ''; @@ -193,10 +189,31 @@ if (ENVIRONMENT_IS_NODE) { if (typeof process == 'undefined' || !process.release || process.release.name !== 'node') throw new Error('not compiled for this environment (did you build to HTML and try to run it not on the web, or set ENVIRONMENT to something - like node - and run it someplace else - like on the web?)'); #endif #endif + // `require()` is no-op in an ESM module, use `createRequire()` to construct + // the require()` function. This is only necessary for multi-environment + // builds, `-sENVIRONMENT=node` emits a static import declaration instead. + // TODO: Swap all `require()`'s with `import()`'s? +#if EXPORT_ES6 && ENVIRONMENT_MAY_BE_WEB + const { createRequire } = await import('module'); + /** @suppress{duplicate} */ + var require = createRequire(import.meta.url); +#endif + // These modules will usually be used on Node.js. Load them eagerly to avoid + // the complexity of lazy-loading. + var fs = require('fs'); + var nodePath = require('path'); + if (ENVIRONMENT_IS_WORKER) { - scriptDirectory = require('path').dirname(scriptDirectory) + '/'; + scriptDirectory = nodePath.dirname(scriptDirectory) + '/'; } else { +#if EXPORT_ES6 + // EXPORT_ES6 + ENVIRONMENT_IS_NODE always requires use of import.meta.url, + // since there's no way getting the current absolute path of the module when + // support for that is not available. + scriptDirectory = require('url').fileURLToPath(new URL('./', import.meta.url)); // includes trailing slash +#else scriptDirectory = __dirname + '/'; +#endif } #include "node_shell_read.js" diff --git a/test/test_other.py b/test/test_other.py index f20c7691d9b64..4459f37ba023b 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -238,41 +238,68 @@ def test_emcc_generate_config(self): self.assertContained('LLVM_ROOT', config_contents) os.remove(config_path) - def test_emcc_output_mjs(self): - self.run_process([EMCC, '-o', 'hello_world.mjs', test_file('hello_world.c')]) - output = read_file('hello_world.mjs') - self.assertContained('export default Module;', output) - # TODO(sbc): Test that this is actually runnable. We currently don't have - # any tests for EXPORT_ES6 but once we do this should be enabled. - # self.assertContained('hello, world!', self.run_js('hello_world.mjs')) + @parameterized({ + '': ([],), + 'node': (['-sENVIRONMENT=node'],), + }) + def test_emcc_output_mjs(self, args): + create_file('extern-post.js', 'await Module();') + self.run_process([EMCC, '-o', 'hello_world.mjs', + '--extern-post-js', 'extern-post.js', + test_file('hello_world.c')] + args) + src = read_file('hello_world.mjs') + self.assertContained('export default Module;', src) + self.assertContained('hello, world!', self.run_js('hello_world.mjs')) @parameterized({ - '': (True, [],), - 'no_import_meta': (False, ['-sUSE_ES6_IMPORT_META=0'],), + '': ([],), + 'node': (['-sENVIRONMENT=node'],), }) - def test_emcc_output_worker_mjs(self, has_import_meta, args): + @node_pthreads + def test_emcc_output_worker_mjs(self, args): + create_file('extern-post.js', 'await Module();') os.mkdir('subdir') - self.run_process([EMCC, '-o', 'subdir/hello_world.mjs', '-pthread', '-O1', + self.run_process([EMCC, '-o', 'subdir/hello_world.mjs', + '-sEXIT_RUNTIME', '-sPROXY_TO_PTHREAD', '-pthread', '-O1', + '--extern-post-js', 'extern-post.js', test_file('hello_world.c')] + args) src = read_file('subdir/hello_world.mjs') - self.assertContainedIf("new URL('hello_world.wasm', import.meta.url)", src, condition=has_import_meta) - self.assertContainedIf("new Worker(new URL('hello_world.worker.js', import.meta.url))", src, condition=has_import_meta) + self.assertContained("new URL('hello_world.wasm', import.meta.url)", src) + self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url))", src) self.assertContained('export default Module;', src) src = read_file('subdir/hello_world.worker.js') self.assertContained('import("./hello_world.mjs")', src) + self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs')) + @node_pthreads def test_emcc_output_worker_mjs_single_file(self): + create_file('extern-post.js', 'await Module();') self.run_process([EMCC, '-o', 'hello_world.mjs', '-pthread', + '--extern-post-js', 'extern-post.js', test_file('hello_world.c'), '-sSINGLE_FILE']) src = read_file('hello_world.mjs') self.assertNotContained("new URL('data:", src) self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url))", src) + self.assertContained('hello, world!', self.run_js('hello_world.mjs')) def test_emcc_output_mjs_closure(self): + create_file('extern-post.js', 'await Module();') self.run_process([EMCC, '-o', 'hello_world.mjs', + '--extern-post-js', 'extern-post.js', test_file('hello_world.c'), '--closure=1']) src = read_file('hello_world.mjs') self.assertContained('new URL("hello_world.wasm", import.meta.url)', src) + self.assertContained('hello, world!', self.run_js('hello_world.mjs')) + + def test_emcc_output_mjs_web_no_import_meta(self): + # Ensure we don't emit import.meta.url at all for: + # ENVIRONMENT=web + EXPORT_ES6 + USE_ES6_IMPORT_META=0 + self.run_process([EMCC, '-o', 'hello_world.mjs', + test_file('hello_world.c'), + '-sENVIRONMENT=web', '-sUSE_ES6_IMPORT_META=0']) + src = read_file('hello_world.mjs') + self.assertNotContained('import.meta.url', src) + self.assertContained('export default Module;', src) def test_export_es6_implies_modularize(self): self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6']) @@ -283,6 +310,11 @@ def test_export_es6_requires_modularize(self): err = self.expect_fail([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6', '-sMODULARIZE=0']) self.assertContained('EXPORT_ES6 requires MODULARIZE to be set', err) + def test_export_es6_node_requires_import_meta(self): + err = self.expect_fail([EMCC, test_file('hello_world.c'), + '-sENVIRONMENT=node', '-sEXPORT_ES6', '-sUSE_ES6_IMPORT_META=0']) + self.assertContained('EXPORT_ES6 and ENVIRONMENT=*node* requires USE_ES6_IMPORT_META to be set', err) + def test_export_es6_allows_export_in_post_js(self): self.run_process([EMCC, test_file('hello_world.c'), '-O3', '-sEXPORT_ES6', '--post-js', test_file('export_module.js')]) src = read_file('a.out.js') diff --git a/third_party/closure-compiler/node-externs/url.js b/third_party/closure-compiler/node-externs/url.js index d3f2878857227..7f848f3fe7f09 100644 --- a/third_party/closure-compiler/node-externs/url.js +++ b/third_party/closure-compiler/node-externs/url.js @@ -61,3 +61,15 @@ url.format = function(urlObj) {}; * @nosideeffects */ url.resolve = function(from, to) {}; + +/** + * @param {url.URL|string} url + * @return {string} + */ +url.fileURLToPath = function(url) {}; + +/** + * @param {string} path + * @return {url.URL} + */ +url.pathToFileURL = function(path) {};