[mypyc] librt base64: support pyodide via the NEON intrinsics (#20316)

Fixes the following error when building lib-rt with pyodide

> ` /home/runner/work/librt/librt/.pyodide_build/pywasmcross_symlinks/cc
-fno-strict-overflow -Wsign-compare -Wunreachable-code -DNDEBUG -g -O3
-Wall -O2 -g0 -fPIC -DPY_CALL_TRAMPOLINE -I. -Ibase64
-I/tmp/build-env-ftmn_jvi/include
-I/home/runner/.cache/cibuildwheel/pyodide-build-0.30.7/0.27.7/xbuildenv/pyodide-root/cpython/installs/python-3.12.7/include/python3.12
-c base64/arch/avx/codec.c -o
build/temp.emscripten_3_1_58_wasm32-cpython-312/base64/arch/avx/codec.o
-O3 -mavx`
> ` emcc: error: passing any of -msse, -msse2, -msse3, -mssse3,
-msse4.1, -msse4.2, -msse4, -mavx, -mfpu=neon flags also requires
passing -msimd128 (or -mrelaxed-simd)!`

Instead of using emscripten's compatibility headers for X86 intrinsics,
this PR has pyodide/emscripten use the NEON intrinsics as they better
match the underlying SIMD capabilities in WASM. And it skips the "CPU
detection" which would not work (nor make sense) in a WASM setting.

Ideally a pure WASM implementation would be added to base64 and
backported here as well.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: mr-c

mypyc/build_setup.py

@@ -1,3 +1,4 @@
+import os
 import platform
 import sys
 
@@ -30,32 +31,41 @@
 
 ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn  # type: ignore[attr-defined]
 X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64")
+PYODIDE = "PYODIDE" in os.environ
 
 
 def spawn(self, cmd, **kwargs) -> None:  # type: ignore[no-untyped-def]
-    compiler_type: str = self.compiler_type
-    extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type]
-    new_cmd = list(cmd)
-    if X86_64 and extra_options is not None:
-        # filenames are closer to the end of command line
+    if PYODIDE:
+        new_cmd = list(cmd)
         for argument in reversed(new_cmd):
-            # Check if the matching argument contains a source filename.
             if not str(argument).endswith(".c"):
                 continue
+            if "base64/arch/" in str(argument):
+                new_cmd.extend(["-msimd128"])
+    else:
+        compiler_type: str = self.compiler_type
+        extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type]
+        new_cmd = list(cmd)
+        if X86_64 and extra_options is not None:
+            # filenames are closer to the end of command line
+            for argument in reversed(new_cmd):
+                # Check if the matching argument contains a source filename.
+                if not str(argument).endswith(".c"):
+                    continue
 
-            for path in extra_options.keys():
-                if path in str(argument):
-                    if compiler_type == "bcpp":
-                        compiler = new_cmd.pop()
-                        # Borland accepts a source file name at the end,
-                        # insert the options before it
-                        new_cmd.extend(extra_options[path])
-                        new_cmd.append(compiler)
-                    else:
-                        new_cmd.extend(extra_options[path])
-
-                    # path component is found, no need to search any further
-                    break
+                for path in extra_options.keys():
+                    if path in str(argument):
+                        if compiler_type == "bcpp":
+                            compiler = new_cmd.pop()
+                            # Borland accepts a source file name at the end,
+                            # insert the options before it
+                            new_cmd.extend(extra_options[path])
+                            new_cmd.append(compiler)
+                        else:
+                            new_cmd.extend(extra_options[path])
+
+                        # path component is found, no need to search any further
+                        break
     self.__spawn(new_cmd, **kwargs)
 
 

mypyc/lib-rt/base64/arch/neon64/codec.c

@@ -12,7 +12,7 @@
 #include <arm_neon.h>
 
 // Only enable inline assembly on supported compilers.
-#if defined(__GNUC__) || defined(__clang__)
+#if !defined(__wasm__) && (defined(__GNUC__) || defined(__clang__))
 #define BASE64_NEON64_USE_ASM
 #endif
 

mypyc/lib-rt/base64/config.h

@@ -13,7 +13,7 @@
 #define BASE64_WITH_NEON32 0
 #define HAVE_NEON32 BASE64_WITH_NEON32
 
-#if defined(__APPLE__) && defined(__aarch64__)
+#if (defined(__APPLE__) && defined(__aarch64__)) || (defined(__wasm__) && defined(__wasm_simd128__))
 #define BASE64_WITH_NEON64 1
 #else
 #define BASE64_WITH_NEON64 0

mypyc/lib-rt/setup.py

@@ -42,32 +42,41 @@
 
 ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn  # type: ignore[attr-defined]
 X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64")
+PYODIDE = "PYODIDE" in os.environ
 
 
 def spawn(self, cmd, **kwargs) -> None:  # type: ignore[no-untyped-def]
-    compiler_type: str = self.compiler_type
-    extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type]
-    new_cmd = list(cmd)
-    if X86_64 and extra_options is not None:
-        # filenames are closer to the end of command line
+    if PYODIDE:
+        new_cmd = list(cmd)
         for argument in reversed(new_cmd):
-            # Check if the matching argument contains a source filename.
             if not str(argument).endswith(".c"):
                 continue
-
-            for path in extra_options.keys():
-                if path in str(argument):
-                    if compiler_type == "bcpp":
-                        compiler = new_cmd.pop()
-                        # Borland accepts a source file name at the end,
-                        # insert the options before it
-                        new_cmd.extend(extra_options[path])
-                        new_cmd.append(compiler)
-                    else:
-                        new_cmd.extend(extra_options[path])
-
-                    # path component is found, no need to search any further
-                    break
+            if "base64/arch/" in str(argument):
+                new_cmd.extend(["-msimd128"])
+    else:
+        compiler_type: str = self.compiler_type
+        extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type]
+        new_cmd = list(cmd)
+        if X86_64 and extra_options is not None:
+            # filenames are closer to the end of command line
+            for argument in reversed(new_cmd):
+                # Check if the matching argument contains a source filename.
+                if not str(argument).endswith(".c"):
+                    continue
+
+                for path in extra_options.keys():
+                    if path in str(argument):
+                        if compiler_type == "bcpp":
+                            compiler = new_cmd.pop()
+                            # Borland accepts a source file name at the end,
+                            # insert the options before it
+                            new_cmd.extend(extra_options[path])
+                            new_cmd.append(compiler)
+                        else:
+                            new_cmd.extend(extra_options[path])
+
+                        # path component is found, no need to search any further
+                        break
     self.__spawn(new_cmd, **kwargs)