Add SHARED_MEMORY support. (#16419)

juj · web-flow · commit a29f6682a579 · 2022-03-04T15:24:27.000Z
* Add SHARED_MEMORY support.

* Add missing SHARED_MEMORY check.

* Add missing SHARED_MEMORY check

* Fix test run
diff --git a/emcc.py b/emcc.py
@@ -308,7 +308,7 @@ def setup_environment_settings():
   if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.PROXY_TO_WORKER:
     exit_with_error('If you specify --proxy-to-worker and specify a "-s ENVIRONMENT=" directive, it must include "worker" as a target! (Try e.g. -s ENVIRONMENT=web,worker)')
 
-  if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.USE_PTHREADS:
+  if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.SHARED_MEMORY:
     exit_with_error('When building with multithreading enabled and a "-s ENVIRONMENT=" directive is specified, it must include "worker" as a target! (Try e.g. -s ENVIRONMENT=web,worker)')
 
 
@@ -830,6 +830,9 @@ def get_cflags(user_args):
   if settings.EMSCRIPTEN_TRACING:
     cflags.append('-D__EMSCRIPTEN_TRACING__=1')
 
+  if settings.SHARED_MEMORY:
+    cflags.append('-D__EMSCRIPTEN_SHARED_MEMORY__=1')
+
   if not settings.STRICT:
     # The preprocessor define EMSCRIPTEN is deprecated. Don't pass it to code
     # in strict mode. Code should use the define __EMSCRIPTEN__ instead.
@@ -1432,7 +1435,10 @@ def phase_setup(options, state, newargs, user_settings):
   if settings.MAIN_MODULE or settings.SIDE_MODULE:
     settings.RELOCATABLE = 1
 
-  if settings.USE_PTHREADS and '-pthread' not in newargs:
+  if settings.USE_PTHREADS:
+    settings.SHARED_MEMORY = 1
+
+  if settings.SHARED_MEMORY and '-pthread' not in newargs:
     newargs += ['-pthread']
 
   if 'DISABLE_EXCEPTION_CATCHING' in user_settings and 'EXCEPTION_CATCHING_ALLOWED' in user_settings:
@@ -1687,7 +1693,7 @@ def phase_linker_setup(options, state, newargs, user_settings):
   # https://github.com/whatwg/encoding/issues/172
   # When supporting shell environments, do not do this as TextDecoder is not
   # widely supported there.
-  if settings.SHRINK_LEVEL >= 2 and not settings.USE_PTHREADS and \
+  if settings.SHRINK_LEVEL >= 2 and not settings.SHARED_MEMORY and \
      not settings.ENVIRONMENT_MAY_BE_SHELL:
     default_setting(user_settings, 'TEXTDECODER', 2)
 
@@ -2214,7 +2220,7 @@ def check_memory_setting(setting):
     settings.LEGALIZE_JS_FFI = 0
 
   # TODO(sbc): Remove WASM2JS here once the size regression it would introduce has been fixed.
-  if settings.USE_PTHREADS or settings.RELOCATABLE or settings.ASYNCIFY_LAZY_LOAD_CODE or settings.WASM2JS:
+  if settings.SHARED_MEMORY or settings.RELOCATABLE or settings.ASYNCIFY_LAZY_LOAD_CODE or settings.WASM2JS:
     settings.IMPORTED_MEMORY = 1
 
   if settings.WASM_BIGINT:
@@ -2247,9 +2253,9 @@ def check_memory_setting(setting):
     # can use a .mem file like asm.js used to.
     # generally we follow what the options tell us to do (which is to use
     # a .mem file in most cases, since it is binary & compact). however, for
-    # pthreads we must keep the memory segments in the wasm as they will be
-    # passive segments which the .mem format cannot handle.
-    settings.MEM_INIT_IN_WASM = not options.memory_init_file or settings.SINGLE_FILE or settings.USE_PTHREADS
+    # shared memory builds we must keep the memory segments in the wasm as
+    # they will be passive segments which the .mem format cannot handle.
+    settings.MEM_INIT_IN_WASM = not options.memory_init_file or settings.SINGLE_FILE or settings.SHARED_MEMORY
   else:
     # wasm includes the mem init in the wasm binary. The exception is
     # wasm2js, which behaves more like js.
diff --git a/src/jsifier.js b/src/jsifier.js
@@ -279,18 +279,20 @@ function ${name}(${args}) {
       if (isFunction) {
         // Emit the body of a JS library function.
         const proxyingMode = LibraryManager.library[ident + '__proxy'];
-        if (USE_PTHREADS && proxyingMode) {
+        if (SHARED_MEMORY && proxyingMode) {
           if (proxyingMode !== 'sync' && proxyingMode !== 'async') {
             throw new Error(`Invalid proxyingMode ${ident}__proxy: '${proxyingMode}' specified!`);
           }
           const sync = proxyingMode === 'sync';
           assert(typeof original == 'function');
-          contentText = modifyFunction(snippet, (name, args, body) => `
+          if (USE_PTHREADS) {
+            contentText = modifyFunction(snippet, (name, args, body) => `
 function ${name}(${args}) {
   if (ENVIRONMENT_IS_PTHREAD)
     return _emscripten_proxy_to_main_thread_js(${proxiedFunctionTable.length}, ${+sync}${args ? ', ' : ''}${args});
   ${body}
 }\n`);
+          }
           proxiedFunctionTable.push(finalName);
         } else if ((USE_ASAN || USE_LSAN || UBSAN_RUNTIME) && LibraryManager.library[ident + '__noleakcheck']) {
           contentText = modifyFunction(snippet, (name, args, body) => `
diff --git a/src/library.js b/src/library.js
@@ -163,9 +163,9 @@ LibraryManager.library = {
     return false; // malloc will report failure
 #endif // ABORTING_MALLOC
 #else // ALLOW_MEMORY_GROWTH == 0
-    // With pthreads, races can happen (another thread might increase the size
+    // With multithreaded builds, races can happen (another thread might increase the size
     // in between), so return a failure, and let the caller retry.
-#if USE_PTHREADS
+#if SHARED_MEMORY
     if (requestedSize <= oldSize) {
       return false;
     }
diff --git a/src/library_exceptions.js b/src/library_exceptions.js
@@ -74,7 +74,7 @@ var LibraryExceptions = {
     }
 
     this.add_ref = function() {
-#if USE_PTHREADS
+#if SHARED_MEMORY
       Atomics.add(HEAP32, (this.ptr + {{{ C_STRUCTS.__cxa_exception.referenceCount }}}) >> 2, 1);
 #else
       var value = {{{ makeGetValue('this.ptr', C_STRUCTS.__cxa_exception.referenceCount, 'i32') }}};
@@ -84,7 +84,7 @@ var LibraryExceptions = {
 
     // Returns true if last reference released.
     this.release_ref = function() {
-#if USE_PTHREADS
+#if SHARED_MEMORY
       var prev = Atomics.sub(HEAP32, (this.ptr + {{{ C_STRUCTS.__cxa_exception.referenceCount }}}) >> 2, 1);
 #else
       var prev = {{{ makeGetValue('this.ptr', C_STRUCTS.__cxa_exception.referenceCount, 'i32') }}};
diff --git a/src/preamble.js b/src/preamble.js
@@ -48,10 +48,10 @@ if (typeof WebAssembly != 'object') {
 
 var wasmMemory;
 
-#if USE_PTHREADS
+#if SHARED_MEMORY
 // For sending to workers.
 var wasmModule;
-#endif // USE_PTHREADS
+#endif // SHARED_MEMORY
 
 //========================================
 // Runtime essentials
@@ -1079,7 +1079,8 @@ function createWasm() {
     // is in charge of programatically exporting them on the global object.
     exportAsmFunctions(exports);
 #endif
-#if USE_PTHREADS
+
+#if USE_PTHREADS// || WASM_WORKERS
     // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers.
     wasmModule = module;
     // Instantiation is synchronous in pthreads and we assert on run dependencies.
@@ -1131,7 +1132,7 @@ function createWasm() {
     assert(Module === trueModule, 'the Module object should not be replaced during async compilation - perhaps the order of HTML elements is wrong?');
     trueModule = null;
 #endif
-#if USE_PTHREADS || RELOCATABLE
+#if SHARED_MEMORY || RELOCATABLE
     receiveInstance(result['instance'], result['module']);
 #else
     // TODO: Due to Closure regression https://github.com/google/closure-compiler/issues/3193, the above line no longer optimizes out down to the following line.
@@ -1252,6 +1253,7 @@ function createWasm() {
   // User shell pages can write their own Module.instantiateWasm = function(imports, successCallback) callback
   // to manually instantiate the Wasm module themselves. This allows pages to run the instantiation parallel
   // to any other async startup actions they are performing.
+  // Also pthreads and wasm workers initialize the wasm instance through this path.
   if (Module['instantiateWasm']) {
 #if USE_OFFSET_CONVERTER
 #if ASSERTIONS && USE_PTHREADS
diff --git a/src/preamble_minimal.js b/src/preamble_minimal.js
@@ -57,7 +57,7 @@ var HEAP8, HEAP16, HEAP32, HEAPU8, HEAPU16, HEAPU32, HEAPF32, HEAPF64,
 
 
 function updateGlobalBufferAndViews(b) {
-#if ASSERTIONS && USE_PTHREADS
+#if ASSERTIONS && SHARED_MEMORY
   assert(b instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
 #endif
   buffer = b;
@@ -84,10 +84,10 @@ if (!ENVIRONMENT_IS_PTHREAD) {
 #endif
   wasmMemory = new WebAssembly.Memory({
     'initial': {{{ INITIAL_MEMORY >>> 16 }}}
-#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != FOUR_GB
+#if SHARED_MEMORY || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != FOUR_GB
     , 'maximum': {{{ (ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != FOUR_GB ? MAXIMUM_MEMORY : INITIAL_MEMORY) >>> 16 }}}
 #endif
-#if USE_PTHREADS
+#if SHARED_MEMORY
     , 'shared': true
 #endif
     });
diff --git a/src/runtime_init_memory.js b/src/runtime_init_memory.js
@@ -34,12 +34,12 @@ if (ENVIRONMENT_IS_PTHREAD) {
 #else
       'maximum': INITIAL_MEMORY / {{{ WASM_PAGE_SIZE }}}
 #endif // ALLOW_MEMORY_GROWTH
-#if USE_PTHREADS
+#if SHARED_MEMORY
       ,
       'shared': true
 #endif
     });
-#if USE_PTHREADS
+#if SHARED_MEMORY
     if (!(wasmMemory.buffer instanceof SharedArrayBuffer)) {
       err('requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
       if (ENVIRONMENT_IS_NODE) {
diff --git a/src/runtime_strings.js b/src/runtime_strings.js
@@ -9,7 +9,7 @@
 // Given a pointer 'ptr' to a null-terminated UTF8-encoded string in the given array that contains uint8 values, returns
 // a copy of that string as a Javascript String object.
 
-#if USE_PTHREADS && TEXTDECODER
+#if SHARED_MEMORY && TEXTDECODER
 /**
  * UTF8Decoder.decode may not work with a view of a SharedArrayBuffer, see
  * https://github.com/whatwg/encoding/issues/172
@@ -36,10 +36,10 @@ function TextDecoderWrapper(encoding) {
 #endif
 
 #if TEXTDECODER == 2
-var UTF8Decoder = new TextDecoder{{{ USE_PTHREADS ? 'Wrapper' : ''}}}('utf8');
+var UTF8Decoder = new TextDecoder{{{ SHARED_MEMORY ? 'Wrapper' : ''}}}('utf8');
 #else // TEXTDECODER == 2
 #if TEXTDECODER
-var UTF8Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder{{{ USE_PTHREADS ? 'Wrapper' : ''}}}('utf8') : undefined;
+var UTF8Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder{{{ SHARED_MEMORY ? 'Wrapper' : ''}}}('utf8') : undefined;
 #endif // TEXTDECODER
 #endif // TEXTDECODER == 2
 
diff --git a/src/runtime_strings_extra.js b/src/runtime_strings_extra.js
@@ -32,10 +32,10 @@ function stringToAscii(str, outPtr) {
 // a copy of that string as a Javascript String object.
 
 #if TEXTDECODER == 2
-var UTF16Decoder = new TextDecoder{{{ USE_PTHREADS ? 'Wrapper' : ''}}}('utf-16le');
+var UTF16Decoder = new TextDecoder{{{ SHARED_MEMORY ? 'Wrapper' : ''}}}('utf-16le');
 #else // TEXTDECODER == 2
 #if TEXTDECODER
-var UTF16Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder{{{ USE_PTHREADS ? 'Wrapper' : ''}}}('utf-16le') : undefined;
+var UTF16Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder{{{ SHARED_MEMORY ? 'Wrapper' : ''}}}('utf-16le') : undefined;
 #endif // TEXTDECODER
 #endif // TEXTDECODER == 2
 
diff --git a/src/settings.js b/src/settings.js
@@ -1465,7 +1465,11 @@ var SDL2_MIXER_FORMATS = ["ogg"];
 // [other]
 var IN_TEST_HARNESS = 0;
 
-// If true, enables support for pthreads.
+// If 1, target compiling a shared Wasm Memory.
+// [compile+link] - affects user code at compile and system libraries at link.
+var SHARED_MEMORY = 0;
+
+// If true, enables support for pthreads. This implies SHARED_MEMORY.
 // This setting is equivalent to `-pthread`, which should be preferred.
 // [compile+link] - affects user code at compile and system libraries at link.
 var USE_PTHREADS = 0;
diff --git a/src/shell.js b/src/shell.js
@@ -121,7 +121,7 @@ if (Module['ENVIRONMENT']) {
 var ENVIRONMENT_IS_PTHREAD = Module['ENVIRONMENT_IS_PTHREAD'] || false;
 #endif
 
-#if USE_PTHREADS && !MODULARIZE
+#if SHARED_MEMORY && !MODULARIZE
 // In MODULARIZE mode _scriptDir needs to be captured already at the very top of the page immediately when the page is parsed, so it is generated there
 // before the page load. In non-MODULARIZE modes generate it here.
 #if EXPORT_ES6
diff --git a/src/wasm2js.js b/src/wasm2js.js
@@ -19,7 +19,7 @@ WebAssembly = {
   // not a fully general polyfill.
   /** @constructor */
   Memory: function(opts) {
-#if USE_PTHREADS
+#if SHARED_MEMORY
     this.buffer = new SharedArrayBuffer(opts['initial'] * {{{ WASM_PAGE_SIZE }}});
 #else
     this.buffer = new ArrayBuffer(opts['initial'] * {{{ WASM_PAGE_SIZE }}});
@@ -74,7 +74,7 @@ WebAssembly = {
       then: function(ok) {
         var module = new WebAssembly.Module(binary);
         ok({
-#if USE_PTHREADS
+#if SHARED_MEMORY
           'module': module,
 #endif
           'instance': new WebAssembly.Instance(module)
diff --git a/tests/common.py b/tests/common.py
@@ -230,6 +230,20 @@ def modified(self):
   return decorated
 
 
+def also_with_minimal_runtime(f):
+  assert callable(f)
+
+  def metafunc(self, with_minimal_runtime):
+    assert self.get_setting('MINIMAL_RUNTIME') is None
+    if with_minimal_runtime:
+      self.set_setting('MINIMAL_RUNTIME', 1)
+    f(self)
+
+  metafunc._parameterize = {'': (False,),
+                            'minimal_runtime': (True,)}
+  return metafunc
+
+
 def ensure_dir(dirname):
   dirname = Path(dirname)
   dirname.mkdir(parents=True, exist_ok=True)
diff --git a/tests/test_other.py b/tests/test_other.py
@@ -35,6 +35,7 @@
 from common import env_modify, no_mac, no_windows, requires_native_clang, with_env_modify
 from common import create_file, parameterized, NON_ZERO, node_pthreads, TEST_ROOT, test_file
 from common import compiler_for, read_file, read_binary, EMBUILDER, require_v8, require_node
+from common import also_with_minimal_runtime
 from tools import shared, building, utils, deps_info, response_file
 import common
 import jsrun
@@ -11771,3 +11772,12 @@ def test_error_in_js_libraries(self):
     self.assertNotContained('This error should not be present!', err)
     self.assertContained('error_in_js_libraries.js:5: #error This is an error string!', err)
     self.assertContained('error_in_js_libraries.js:7: #error This is a second error string!', err)
+
+  # Tests building with -sSHARED_MEMORY
+  @also_with_minimal_runtime
+  def test_shared_memory(self):
+    self.do_runf(test_file('wasm_worker/shared_memory.c'), '0', emcc_args=[])
+    self.node_args += ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory']
+    self.do_runf(test_file('wasm_worker/shared_memory.c'), '1', emcc_args=['-sSHARED_MEMORY'])
+#    self.do_runf(test_file('wasm_worker/shared_memory.c'), '1', emcc_args=['-sWASM_WORKERS'])
+    self.do_runf(test_file('wasm_worker/shared_memory.c'), '1', emcc_args=['-pthread'])
diff --git a/tests/wasm_worker/shared_memory.c b/tests/wasm_worker/shared_memory.c
@@ -0,0 +1,7 @@
+#include <emscripten.h>
+#include <stdio.h>
+
+int main()
+{
+	printf("%d\n", EM_ASM_INT(return buffer instanceof SharedArrayBuffer));
+}
diff --git a/tools/building.py b/tools/building.py
@@ -287,7 +287,7 @@ def lld_flags_for_executable(external_symbols):
   if settings.IMPORTED_MEMORY:
     cmd.append('--import-memory')
 
-  if settings.USE_PTHREADS:
+  if settings.SHARED_MEMORY:
     cmd.append('--shared-memory')
 
   if settings.MEMORY64:
diff --git a/tools/settings.py b/tools/settings.py
@@ -62,6 +62,7 @@
     'STRICT',
     'EMSCRIPTEN_TRACING',
     'USE_PTHREADS',
+    'SHARED_MEMORY',
     'SUPPORT_LONGJMP',
     'DEFAULT_TO_CXX',
     'WASM_OBJECT_FILES',
diff --git a/tools/system_libs.py b/tools/system_libs.py
@@ -1084,7 +1084,7 @@ def get_ext(self):
     return '.o'
 
   def can_use(self):
-    return super().can_use() and settings.USE_PTHREADS
+    return super().can_use() and settings.SHARED_MEMORY
 
 
 class libcxxabi(NoExceptLibrary, MTLibrary):
@@ -1717,7 +1717,7 @@ def add_library(libname):
     libs_to_link.append((lib.get_link_flag(), need_whole_archive))
 
   if '-nostartfiles' not in args:
-    if settings.USE_PTHREADS:
+    if settings.SHARED_MEMORY:
       add_library('crtbegin')
 
     if settings.STANDALONE_WASM:
