Use stdin to pass settings to compiler.mjs / preprocessor.mjs (emscripten-core#23650)

This avoids writing more files to disc than we need to.  One of my
hopes here is that this will be a little more performant, especially
on windows, where the filesystem isn't the fastest.

In order to aid debugging we stash copied of the these generated
settings using the save_intermediate mechanism.

Author: sbc100

test/test_other.py

@@ -2969,7 +2969,7 @@ def test_emcc_debug_files(self, opt):
           self.assertFalse(os.path.exists(self.canonical_temp_dir))
         else:
           print(sorted(os.listdir(self.canonical_temp_dir)))
-          self.assertExists(os.path.join(self.canonical_temp_dir, 'emcc-02-original.js'))
+          self.assertExists(os.path.join(self.canonical_temp_dir, 'emcc-04-original.js'))
 
   def test_debuginfo_line_tables_only(self):
     def test(do_compile):

tools/building.py

@@ -1224,16 +1224,45 @@ def run_wasm_opt(infile, outfile=None, args=[], **kwargs):  # noqa
   return run_binaryen_command('wasm-opt', infile, outfile, args=args, **kwargs)
 
 
-def save_intermediate(src, dst):
+intermediate_counter = 0
+
+
+def new_intermediate_filename(name):
+  assert DEBUG
+  global intermediate_counter
+  basename = 'emcc-%02d-%s' % (intermediate_counter, name)
+  intermediate_counter += 1
+  filename = os.path.join(shared.CANONICAL_TEMP_DIR, basename)
+  logger.debug('saving intermediate file %s' % filename)
+  return filename
+
+
+def save_intermediate(src, name):
+  """Copy an existing file CANONICAL_TEMP_DIR"""
   if DEBUG:
-    dst = 'emcc-%02d-%s' % (save_intermediate.counter, dst)
-    save_intermediate.counter += 1
-    dst = os.path.join(shared.CANONICAL_TEMP_DIR, dst)
-    logger.debug('saving debug copy %s' % dst)
-    shutil.copyfile(src, dst)
+    shutil.copyfile(src, new_intermediate_filename(name))
+
+
+def write_intermediate(content, name):
+  """Generate a new debug file CANONICAL_TEMP_DIR"""
+  if DEBUG:
+    utils.write_file(new_intermediate_filename(name), content)
+
+
+def read_and_preprocess(filename, expand_macros=False):
+  # Create a settings file with the current settings to pass to the JS preprocessor
+  settings_json = json.dumps(settings.external_dict(), sort_keys=True, indent=2)
+  write_intermediate(settings_json, 'settings.json')
 
+  # Run the JS preprocessor
+  dirname, filename = os.path.split(filename)
+  if not dirname:
+    dirname = None
+  args = ['-', filename]
+  if expand_macros:
+    args += ['--expand-macros']
 
-save_intermediate.counter = 0  # type: ignore
+  return shared.run_js_tool(path_from_root('tools/preprocessor.mjs'), args, input=settings_json, stdout=subprocess.PIPE, cwd=dirname)
 
 
 def js_legalization_pass_flags():

tools/compiler.mjs

@@ -37,8 +37,12 @@ Usage: compiler.mjs <settings.json> [-o out.js] [--symbols-only]`);
 }
 
 // Load settings from JSON passed on the command line
-const settingsFile = positionals[0];
+let settingsFile = positionals[0];
 assert(settingsFile, 'settings file not specified');
+if (settingsFile == '-') {
+  // Read settings json from stdin (FD 0)
+  settingsFile = 0;
+}
 const userSettings = JSON.parse(readFile(settingsFile));
 applySettings(userSettings);
 

tools/emscripten.py

@@ -182,16 +182,15 @@ def compile_javascript(symbols_only=False):
     stderr_file = open(stderr_file, 'w')
 
   # Save settings to a file to work around v8 issue 1579
-  with shared.get_temp_files().get_file('.json') as settings_file:
-    with open(settings_file, 'w') as s:
-      json.dump(settings.external_dict(), s, sort_keys=True, indent=2)
-
-    # Call js compiler
-    args = [settings_file]
-    if symbols_only:
-      args += ['--symbols-only']
-    out = shared.run_js_tool(path_from_root('tools/compiler.mjs'),
-                             args, stdout=subprocess.PIPE, stderr=stderr_file, encoding='utf-8')
+  settings_json = json.dumps(settings.external_dict(), sort_keys=True, indent=2)
+  building.write_intermediate(settings_json, 'settings.json')
+
+  # Call js compiler. Passing `-` here mean read the settings from stdin.
+  args = ['-']
+  if symbols_only:
+    args += ['--symbols-only']
+  out = shared.run_js_tool(path_from_root('tools/compiler.mjs'),
+                           args, input=settings_json, stdout=subprocess.PIPE, stderr=stderr_file)
   if symbols_only:
     glue = None
     forwarded_data = out

tools/link.py

@@ -303,7 +303,7 @@ def read_js_files(files):
   for f in files:
     content = read_file(f)
     if content.startswith('#preprocess\n'):
-      contents.append(shared.read_and_preprocess(f, expand_macros=True))
+      contents.append(building.read_and_preprocess(f, expand_macros=True))
     else:
       contents.append(content)
   contents = '\n'.join(contents)
@@ -2078,7 +2078,7 @@ def fix_es6_import_statements(js_file):
 def create_worker_file(input_file, target_dir, output_file, options):
   output_file = os.path.join(target_dir, output_file)
   input_file = utils.path_from_root(input_file)
-  contents = shared.read_and_preprocess(input_file, expand_macros=True)
+  contents = building.read_and_preprocess(input_file, expand_macros=True)
   write_file(output_file, contents)
 
   fix_es6_import_statements(output_file)
@@ -2284,7 +2284,7 @@ def phase_binaryen(target, options, wasm_target):
     if settings.WASM == 2:
       # With normal wasm2js mode this file gets included as part of the
       # preamble, but with WASM=2 its a separate file.
-      wasm2js_polyfill = shared.read_and_preprocess(utils.path_from_root('src/wasm2js.js'), expand_macros=True)
+      wasm2js_polyfill = building.read_and_preprocess(utils.path_from_root('src/wasm2js.js'), expand_macros=True)
       wasm2js_template = wasm_target + '.js'
       write_file(wasm2js_template, wasm2js_polyfill)
       # generate secondary file for JS symbols
@@ -2543,7 +2543,7 @@ def generate_traditional_runtime_html(target, options, js_target, target_basenam
     # name, but the normal one does not support that currently
     exit_with_error('customizing EXPORT_NAME requires that the HTML be customized to use that name (see https://github.com/emscripten-core/emscripten/issues/10086)')
 
-  shell = shared.read_and_preprocess(options.shell_path)
+  shell = building.read_and_preprocess(options.shell_path)
   if '{{{ SCRIPT }}}' not in shell:
     exit_with_error('HTML shell must contain {{{ SCRIPT }}}, see src/shell.html for an example')
   base_js_target = os.path.basename(js_target)
@@ -2689,7 +2689,7 @@ def generate_worker_js(target, options, js_target, target_basename):
 
 def worker_js_script(proxy_worker_filename):
   web_gl_client_src = read_file(utils.path_from_root('src/webGLClient.js'))
-  proxy_client_src = shared.read_and_preprocess(utils.path_from_root('src/proxyClient.js'), expand_macros=True)
+  proxy_client_src = building.read_and_preprocess(utils.path_from_root('src/proxyClient.js'), expand_macros=True)
   if not settings.SINGLE_FILE and not os.path.dirname(proxy_worker_filename):
     proxy_worker_filename = './' + proxy_worker_filename
   proxy_client_src = do_replace(proxy_client_src, '<<< filename >>>', proxy_worker_filename)

tools/minimal_runtime_shell.py

@@ -7,6 +7,7 @@
 __rootdir__ = os.path.dirname(__scriptdir__)
 sys.path.insert(0, __rootdir__)
 
+from . import building
 from . import shared
 from . import utils
 from . import feature_matrix
@@ -194,7 +195,7 @@ def generate_minimal_runtime_html(target, options, js_target, target_basename):
   temp_files = shared.get_temp_files()
   with temp_files.get_file(suffix='.js') as shell_temp:
     utils.write_file(shell_temp, shell)
-    shell = shared.read_and_preprocess(shell_temp)
+    shell = building.read_and_preprocess(shell_temp)
 
   if re.search(r'{{{\s*SCRIPT\s*}}}', shell):
     shared.exit_with_error('--shell-file "' + options.shell_path + '": MINIMAL_RUNTIME uses a different kind of HTML page shell file than the traditional runtime! Please see $EMSCRIPTEN/src/shell_minimal_runtime.html for a template to use as a basis.')

tools/preprocessor.mjs

@@ -36,8 +36,12 @@ loadDefaultSettings();
 assert(positionals.length == 2, 'Script requires 2 arguments');
 
 // Load settings from JSON passed on the command line
-const settingsFile = positionals[0];
+let settingsFile = positionals[0];
 assert(settingsFile, 'settings file not specified');
+if (settingsFile == '-') {
+  // Read settings json from stdin (FD 0)
+  settingsFile = 0;
+}
 const userSettings = JSON.parse(readFile(settingsFile));
 applySettings(userSettings);
 

tools/shared.py

@@ -8,7 +8,6 @@
 from enum import Enum, unique, auto
 from subprocess import PIPE
 import atexit
-import json
 import logging
 import os
 import re
@@ -719,23 +718,6 @@ def safe_copy(src, dst):
   make_writable(dst)
 
 
-def read_and_preprocess(filename, expand_macros=False):
-  # Create a settings file with the current settings to pass to the JS preprocessor
-  with get_temp_files().get_file('.json') as settings_file:
-    with open(settings_file, 'w') as s:
-      json.dump(settings.external_dict(), s, sort_keys=True, indent=2)
-
-    # Run the JS preprocessor
-    dirname, filename = os.path.split(filename)
-    if not dirname:
-      dirname = None
-    args = [settings_file, filename]
-    if expand_macros:
-      args += ['--expand-macros']
-
-    return run_js_tool(path_from_root('tools/preprocessor.mjs'), args, stdout=subprocess.PIPE, cwd=dirname)
-
-
 def do_replace(input_, pattern, replacement):
   if pattern not in input_:
     exit_with_error('expected to find pattern in input JS: %s' % pattern)