[mypyc] Enable partial, unsafe support for free-threading (#19167)

Enable multi-phase init when using a free-threaded (no-GIL) CPython
build so we can enable proper multihreading.

Work on mypyc/mypyc#1104. Work on mypyc/mypyc#1038.

The implementation is still quite incomplete. We are missing
synchronization in various places, so race conditions can cause
segfaults. Only single-module compilation units are supported for now.

Here's a toy benchmark I used to check that free threading works and can
improve performance:
```
import sys
import threading
import time

def fib(n: int) -> int:
    if n <= 1:
        return n
    else:
        return fib(n - 1) + fib(n - 2)

NTHREADS = 6
print(f"Using {NTHREADS} threads")
print(f"{sys._is_gil_enabled()=}")

t0 = time.time()

threads = []
for i in range(NTHREADS):
    t = threading.Thread(target=lambda: fib(36))
    t.start()
    threads.append(t)

for t in threads:
    t.join()

print()
print('elapsed time:', time.time() - t0)
```

Author: JukkaL

mypyc/codegen/emitmodule.py

@@ -7,6 +7,7 @@
 
 import json
 import os
+import sys
 from collections.abc import Iterable
 from typing import Optional, TypeVar
 
@@ -38,6 +39,7 @@
 )
 from mypyc.codegen.literals import Literals
 from mypyc.common import (
+    IS_FREE_THREADED,
     MODULE_PREFIX,
     PREFIX,
     RUNTIME_C_FILES,
@@ -513,6 +515,9 @@ def __init__(
         self.use_shared_lib = group_name is not None
         self.compiler_options = compiler_options
         self.multi_file = compiler_options.multi_file
+        # Multi-phase init is needed to enable free-threading. In the future we'll
+        # probably want to enable it always, but we'll wait until it's stable.
+        self.multi_phase_init = IS_FREE_THREADED
 
     @property
     def group_suffix(self) -> str:
@@ -869,10 +874,31 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module
         """Emit the PyModuleDef struct for a module and the module init function."""
         module_prefix = emitter.names.private_name(module_name)
         self.emit_module_exec_func(emitter, module_name, module_prefix, module)
+        if self.multi_phase_init:
+            self.emit_module_def_slots(emitter, module_prefix)
         self.emit_module_methods(emitter, module_name, module_prefix, module)
         self.emit_module_def_struct(emitter, module_name, module_prefix)
         self.emit_module_init_func(emitter, module_name, module_prefix)
 
+    def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None:
+        name = f"{module_prefix}_slots"
+        exec_name = f"{module_prefix}_exec"
+
+        emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{")
+        emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},")
+        if sys.version_info >= (3, 12):
+            # Multiple interpreter support requires not using any C global state,
+            # which we don't support yet.
+            emitter.emit_line(
+                "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},"
+            )
+        if sys.version_info >= (3, 13):
+            # Declare support for free-threading to enable experimentation,
+            # even if we don't properly support it.
+            emitter.emit_line("{Py_mod_gil, Py_MOD_GIL_NOT_USED},")
+        emitter.emit_line("{0, NULL},")
+        emitter.emit_line("};")
+
     def emit_module_methods(
         self, emitter: Emitter, module_name: str, module_prefix: str, module: ModuleIR
     ) -> None:
@@ -905,11 +931,15 @@ def emit_module_def_struct(
             "PyModuleDef_HEAD_INIT,",
             f'"{module_name}",',
             "NULL, /* docstring */",
-            "-1,       /* size of per-interpreter state of the module,",
-            "             or -1 if the module keeps state in global variables. */",
-            f"{module_prefix}module_methods",
-            "};",
+            "0,       /* size of per-interpreter state of the module */",
+            f"{module_prefix}module_methods,",
         )
+        if self.multi_phase_init:
+            slots_name = f"{module_prefix}_slots"
+            emitter.emit_line(f"{slots_name}, /* m_slots */")
+        else:
+            emitter.emit_line("NULL,")
+        emitter.emit_line("};")
         emitter.emit_line()
 
     def emit_module_exec_func(
@@ -927,6 +957,8 @@ def emit_module_exec_func(
         module_static = self.module_internal_static_name(module_name, emitter)
         emitter.emit_lines(declaration, "{")
         emitter.emit_line("PyObject* modname = NULL;")
+        if self.multi_phase_init:
+            emitter.emit_line(f"{module_static} = module;")
         emitter.emit_line(
             f'modname = PyObject_GetAttrString((PyObject *){module_static}, "__name__");'
         )
@@ -958,7 +990,10 @@ def emit_module_exec_func(
 
         emitter.emit_line("return 0;")
         emitter.emit_lines("fail:")
-        emitter.emit_lines(f"Py_CLEAR({module_static});", "Py_CLEAR(modname);")
+        if self.multi_phase_init:
+            emitter.emit_lines(f"{module_static} = NULL;", "Py_CLEAR(modname);")
+        else:
+            emitter.emit_lines(f"Py_CLEAR({module_static});", "Py_CLEAR(modname);")
         for name, typ in module.final_names:
             static_name = emitter.static_name(name, module_name)
             emitter.emit_dec_ref(static_name, typ, is_xdec=True)
@@ -980,6 +1015,12 @@ def emit_module_init_func(
             declaration = f"PyObject *CPyInit_{exported_name(module_name)}(void)"
         emitter.emit_lines(declaration, "{")
 
+        if self.multi_phase_init:
+            def_name = f"{module_prefix}module"
+            emitter.emit_line(f"return PyModuleDef_Init(&{def_name});")
+            emitter.emit_line("}")
+            return
+
         exec_func = f"{module_prefix}_exec"
 
         # Store the module reference in a static and return it when necessary.

mypyc/common.py

@@ -88,6 +88,10 @@
 # some details in the PEP are out of date.
 HAVE_IMMORTAL: Final = sys.version_info >= (3, 12)
 
+# Are we running on a free-threaded build (GIL disabled)? This implies that
+# we are on Python 3.13 or later.
+IS_FREE_THREADED: Final = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
+
 
 JsonDict = dict[str, Any]