Add a new way to mark async imports in library files. (#19093)

* Add a new way to mark async imports in library files.

Allow library functions to be asyncify'd with either an `async` function or adding
`$name_async`.

* File extension.

* Fix test.

Author: brendandahl

emcc.py

@@ -101,7 +101,7 @@
 }
 
 DEFAULT_ASYNCIFY_IMPORTS = [
-  'emscripten_sleep', 'emscripten_wget', 'emscripten_wget_data', 'emscripten_idb_load',
+  'emscripten_wget', 'emscripten_wget_data', 'emscripten_idb_load',
   'emscripten_idb_store', 'emscripten_idb_delete', 'emscripten_idb_exists',
   'emscripten_idb_load_blob', 'emscripten_idb_store_blob', 'SDL_Delay',
   'emscripten_scan_registers', 'emscripten_lazy_load_code',
@@ -507,7 +507,7 @@ def ensure_archive_index(archive_file):
     run_process([shared.LLVM_RANLIB, archive_file])
 
 
-def generate_js_symbols():
+def generate_js_sym_info():
   # Runs the js compiler to generate a list of all symbols available in the JS
   # libraries.  This must be done separately for each linker invokation since the
   # list of symbols depends on what settings are used.
@@ -520,11 +520,11 @@ def generate_js_symbols():
 
 
 @ToolchainProfiler.profile_block('JS symbol generation')
-def get_all_js_syms():
+def get_js_sym_info():
   # Avoiding using the cache when generating struct info since
   # this step is performed while the cache is locked.
   if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE:
-    return generate_js_symbols()
+    return generate_js_sym_info()
 
   # We define a cache hit as when the settings and `--js-library` contents are
   # identical.
@@ -545,29 +545,16 @@ def get_all_js_syms():
   def build_symbol_list(filename):
     """Only called when there is no existing symbol list for a given content hash.
     """
-    library_syms = generate_js_symbols()
-    lines = []
+    library_syms = generate_js_sym_info()
 
-    for name, deps in library_syms.items():
-      if deps:
-        lines.append('%s: %s' % (name, ','.join(deps)))
-      else:
-        lines.append(name)
-    write_file(filename, '\n'.join(lines) + '\n')
+    write_file(filename, json.dumps(library_syms, separators=(',', ':')))
 
   # We need to use a separate lock here for symbol lists because, unlike with system libraries,
   # it's normally for these file to get pruned as part of normal operation.  This means that it
   # can be deleted between the `cache.get()` then the `read_file`.
   with filelock.FileLock(cache.get_path(cache.get_path('symbol_lists.lock'))):
-    filename = cache.get(f'symbol_lists/{content_hash}.txt', build_symbol_list)
-    lines = read_file(filename).splitlines()
-    library_syms = {}
-    for line in lines:
-      if ':' in line:
-        name, deps = line.split(':')
-        library_syms[name] = deps.strip().split(',')
-      else:
-        library_syms[line] = []
+    filename = cache.get(f'symbol_lists/{content_hash}.json', build_symbol_list)
+    library_syms = json.loads(read_file(filename))
 
     # Limit of the overall size of the cache to 100 files.
     # This code will get test coverage since a full test run of `other` or `core`
@@ -1334,9 +1321,13 @@ def run(args):
     return 0
 
   js_syms = {}
-  if not settings.SIDE_MODULE:
-    js_syms = get_all_js_syms()
-    deps_info.append_deps_info(js_syms)
+  if not settings.SIDE_MODULE or settings.ASYNCIFY:
+    js_info = get_js_sym_info()
+    if not settings.SIDE_MODULE:
+      js_syms = js_info['deps']
+      deps_info.append_deps_info(js_syms)
+    if settings.ASYNCIFY:
+      settings.ASYNCIFY_IMPORTS += ['env.' + x for x in js_info['asyncFuncs']]
 
   phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs)
 

src/compiler.js

@@ -74,7 +74,7 @@ if (symbolsOnly) {
 }
 
 // Side modules are pure wasm and have no JS
-assert(!SIDE_MODULE, 'JS compiler should not run on side modules');
+assert(!SIDE_MODULE || (ASYNCIFY && global.symbolsOnly), 'JS compiler should only run on side modules if asyncify is used.');
 
 // Output some info and warnings based on settings
 

src/jsifier.js

@@ -69,6 +69,7 @@ function isDefined(symName) {
 function runJSify() {
   const libraryItems = [];
   const symbolDeps = {};
+  const asyncFuncs = [];
   let postSets = [];
 
   LibraryManager.load();
@@ -247,6 +248,18 @@ function ${name}(${args}) {
 
       const deps = LibraryManager.library[symbol + '__deps'] || [];
 
+      let isAsyncFunction = false;
+      if (ASYNCIFY) {
+        const original = LibraryManager.library[symbol];
+        if (typeof original == 'function' ) {
+          isAsyncFunction = LibraryManager.library[symbol + '__async'] ||
+                            original.constructor.name == 'AsyncFunction'
+        }
+        if (isAsyncFunction) {
+          asyncFuncs.push(symbol);
+        }
+      }
+
       if (symbolsOnly) {
         if (!isJsOnlySymbol(symbol) && LibraryManager.library.hasOwnProperty(symbol)) {
           externalDeps = deps.filter((d) => !isJsOnlySymbol(d) && !(d in LibraryManager.library) && typeof d === 'string');
@@ -423,6 +436,9 @@ function ${name}(${args}) {
       if (sig && (RELOCATABLE || ASYNCIFY == 2)) {
         contentText += `\n${mangled}.sig = '${sig}';`;
       }
+      if (ASYNCIFY && isAsyncFunction) {
+        contentText += `\n${mangled}.isAsync = true;`;
+      }
       if (isStub) {
         contentText += `\n${mangled}.stub = true;`;
         if (ASYNCIFY) {
@@ -529,6 +545,7 @@ function ${name}(${args}) {
     print('//FORWARDED_DATA:' + JSON.stringify({
       librarySymbols: librarySymbols,
       warnings: warnings,
+      asyncFuncs,
       ATINITS: ATINITS.join('\n'),
       ATMAINS: ATMAINS.join('\n'),
       ATEXITS: ATEXITS.join('\n'),
@@ -540,7 +557,10 @@ function ${name}(${args}) {
   }
 
   if (symbolsOnly) {
-    print(JSON.stringify(symbolDeps));
+    print(JSON.stringify({
+      deps: symbolDeps,
+      asyncFuncs
+    }));
     return;
   }
 

src/library_async.js

@@ -40,7 +40,8 @@ mergeInto(LibraryManager.library, {
           var original = imports[x];
           var sig = original.sig;
           if (typeof original == 'function') {
-            var isAsyncifyImport = ASYNCIFY_IMPORTS.indexOf(x) >= 0 ||
+            var isAsyncifyImport = original.isAsync ||
+                                   ASYNCIFY_IMPORTS.indexOf(x) >= 0 ||
                                    x.startsWith('__asyncjs__');
 #if ASYNCIFY == 2
             // Wrap async imports with a suspending WebAssembly function.
@@ -451,6 +452,7 @@ mergeInto(LibraryManager.library, {
   },
 
   emscripten_sleep__deps: ['$safeSetTimeout'],
+  emscripten_sleep__async: true,
   emscripten_sleep: function(ms) {
     // emscripten_sleep() does not return a value, but we still need a |return|
     // here for stack switching support (ASYNCIFY=2). In that mode this function

src/utility.js

@@ -182,6 +182,7 @@ function isJsLibraryConfigIdentifier(ident) {
     '__noleakcheck',
     '__internal',
     '__user',
+    '__async',
   ];
   return suffixes.some((suffix) => ident.endsWith(suffix));
 }

tools/gen_sig_info.py

@@ -248,7 +248,7 @@ def extract_sig_info(sig_info, extra_settings=None, extra_cflags=None):
     output = shared.run_js_tool(utils.path_from_root('src/compiler.js'),
                                 ['--symbols-only', settings_json],
                                 stdout=subprocess.PIPE, cwd=utils.path_from_root())
-  symbols = json.loads(output).keys()
+  symbols = json.loads(output)['deps'].keys()
   symbols = [s for s in symbols if not ignore_symbol(s)]
   with tempfiles.get_file('.c') as c_file:
     create_c_file(c_file, symbols)