Rename native_internal to librt.internal (#20014)

Also adjust build/test logic slightly to prepare for more modules in
`librt`. This PR should fix #20006

Author: ilevkivskyi

mypy-requirements.txt

@@ -4,4 +4,4 @@ typing_extensions>=4.6.0
 mypy_extensions>=1.0.0
 pathspec>=0.9.0
 tomli>=1.1.0; python_version<'3.11'
-librt==0.1.1
+librt>=0.2.1

mypy/cache.py

@@ -3,8 +3,7 @@
 from collections.abc import Sequence
 from typing import Final
 
-from mypy_extensions import u8
-from native_internal import (
+from librt.internal import (
     Buffer as Buffer,
     read_bool as read_bool,
     read_float as read_float,
@@ -17,6 +16,7 @@
     write_str as write_str,
     write_tag as write_tag,
 )
+from mypy_extensions import u8
 
 # Always use this type alias to refer to type tags.
 Tag = u8

mypy/typeshed/stubs/librt/METADATA.toml

@@ -1 +1 @@
-version = "0.1.*"
+version = "0.2.*"

mypy/typeshed/stubs/librt/librt/__init__.pyi

@@ -42,6 +42,8 @@
 from mypyc.namegen import exported_name
 from mypyc.options import CompilerOptions
 
+LIBRT_MODULES = [("librt.internal", "librt_internal.c")]
+
 try:
     # Import setuptools so that it monkey-patch overrides distutils
     import setuptools
@@ -492,8 +494,8 @@ def mypycify(
     strict_dunder_typing: bool = False,
     group_name: str | None = None,
     log_trace: bool = False,
-    depends_on_native_internal: bool = False,
-    install_native_libs: bool = False,
+    depends_on_librt_internal: bool = False,
+    install_librt: bool = False,
 ) -> list[Extension]:
     """Main entry point to building using mypyc.
 
@@ -544,11 +546,11 @@ def mypycify(
                    mypyc_trace.txt (derived from executed operations). This is
                    useful for performance analysis, such as analyzing which
                    primitive ops are used the most and on which lines.
-        depends_on_native_internal: This is True only for mypy itself.
-        install_native_libs: If True, also build the native extension modules. Normally,
-                             those are build and published on PyPI separately, but during
-                             tests, we want to use their development versions (i.e. from
-                             current commit).
+        depends_on_librt_internal: This is True only for mypy itself.
+        install_librt: If True, also build the librt extension modules. Normally,
+                       those are build and published on PyPI separately, but during
+                       tests, we want to use their development versions (i.e. from
+                       current commit).
     """
 
     # Figure out our configuration
@@ -562,7 +564,7 @@ def mypycify(
         strict_dunder_typing=strict_dunder_typing,
         group_name=group_name,
         log_trace=log_trace,
-        depends_on_native_internal=depends_on_native_internal,
+        depends_on_librt_internal=depends_on_librt_internal,
     )
 
     # Generate all the actual important C code
@@ -661,21 +663,25 @@ def mypycify(
                 build_single_module(group_sources, cfilenames + shared_cfilenames, cflags)
             )
 
-    if install_native_libs:
-        for name in ["native_internal.c"] + RUNTIME_C_FILES:
+    if install_librt:
+        os.makedirs("librt", exist_ok=True)
+        for name in RUNTIME_C_FILES:
             rt_file = os.path.join(build_dir, name)
             with open(os.path.join(include_dir(), name), encoding="utf-8") as f:
                 write_file(rt_file, f.read())
-        extensions.append(
-            get_extension()(
-                "native_internal",
-                sources=[
-                    os.path.join(build_dir, file)
-                    for file in ["native_internal.c"] + RUNTIME_C_FILES
-                ],
-                include_dirs=[include_dir()],
-                extra_compile_args=cflags,
+        for mod, file_name in LIBRT_MODULES:
+            rt_file = os.path.join(build_dir, file_name)
+            with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f:
+                write_file(rt_file, f.read())
+            extensions.append(
+                get_extension()(
+                    mod,
+                    sources=[
+                        os.path.join(build_dir, file) for file in [file_name] + RUNTIME_C_FILES
+                    ],
+                    include_dirs=[include_dir()],
+                    extra_compile_args=cflags,
+                )
             )
-        )
 
     return extensions

mypy/typeshed/stubs/librt/native_internal.pyi renamed to mypy/typeshed/stubs/librt/librt/internal.pyi

@@ -601,12 +601,12 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
         ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H")
         ext_declarations.emit_line("#include <Python.h>")
         ext_declarations.emit_line("#include <CPy.h>")
-        if self.compiler_options.depends_on_native_internal:
-            ext_declarations.emit_line("#include <native_internal.h>")
+        if self.compiler_options.depends_on_librt_internal:
+            ext_declarations.emit_line("#include <librt_internal.h>")
 
         declarations = Emitter(self.context)
-        declarations.emit_line(f"#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H")
-        declarations.emit_line(f"#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H")
+        declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H")
+        declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H")
         declarations.emit_line("#include <Python.h>")
         declarations.emit_line("#include <CPy.h>")
         declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"')
@@ -1029,8 +1029,8 @@ def emit_module_exec_func(
         declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)"
         module_static = self.module_internal_static_name(module_name, emitter)
         emitter.emit_lines(declaration, "{")
-        if self.compiler_options.depends_on_native_internal:
-            emitter.emit_line("if (import_native_internal() < 0) {")
+        if self.compiler_options.depends_on_librt_internal:
+            emitter.emit_line("if (import_librt_internal() < 0) {")
             emitter.emit_line("return -1;")
             emitter.emit_line("}")
         emitter.emit_line("PyObject* modname = NULL;")

mypyc/build.py

@@ -514,7 +514,7 @@ def __hash__(self) -> int:
 
 KNOWN_NATIVE_TYPES: Final = {
     name: RPrimitive(name, is_unboxed=False, is_refcounted=True)
-    for name in ["native_internal.Buffer"]
+    for name in ["librt.internal.Buffer"]
 }
 
 

mypyc/codegen/emitmodule.py

@@ -2,8 +2,8 @@
 #include <Python.h>
 #include <stdint.h>
 #include "CPy.h"
-#define NATIVE_INTERNAL_MODULE
-#include "native_internal.h"
+#define LIBRT_INTERNAL_MODULE
+#include "librt_internal.h"
 
 #define START_SIZE 512
 #define MAX_SHORT_INT_TAGGED (255 << 1)
@@ -558,7 +558,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames
     return Py_None;
 }
 
-static PyMethodDef native_internal_module_methods[] = {
+static PyMethodDef librt_internal_module_methods[] = {
     {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")},
     {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")},
     {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")},
@@ -574,11 +574,11 @@ static PyMethodDef native_internal_module_methods[] = {
 
 static int
 NativeInternal_ABI_Version(void) {
-    return NATIVE_INTERNAL_ABI_VERSION;
+    return LIBRT_INTERNAL_ABI_VERSION;
 }
 
 static int
-native_internal_module_exec(PyObject *m)
+librt_internal_module_exec(PyObject *m)
 {
     if (PyType_Ready(&BufferType) < 0) {
         return -1;
@@ -604,32 +604,32 @@ native_internal_module_exec(PyObject *m)
         (void *)read_tag_internal,
         (void *)NativeInternal_ABI_Version,
     };
-    PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "native_internal._C_API", NULL);
+    PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL);
     if (PyModule_Add(m, "_C_API", c_api_object) < 0) {
         return -1;
     }
     return 0;
 }
 
-static PyModuleDef_Slot native_internal_module_slots[] = {
-    {Py_mod_exec, native_internal_module_exec},
+static PyModuleDef_Slot librt_internal_module_slots[] = {
+    {Py_mod_exec, librt_internal_module_exec},
 #ifdef Py_MOD_GIL_NOT_USED
     {Py_mod_gil, Py_MOD_GIL_NOT_USED},
 #endif
     {0, NULL}
 };
 
-static PyModuleDef native_internal_module = {
+static PyModuleDef librt_internal_module = {
     .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = "native_internal",
+    .m_name = "internal",
     .m_doc = "Mypy cache serialization utils",
     .m_size = 0,
-    .m_methods = native_internal_module_methods,
-    .m_slots = native_internal_module_slots,
+    .m_methods = librt_internal_module_methods,
+    .m_slots = librt_internal_module_slots,
 };
 
 PyMODINIT_FUNC
-PyInit_native_internal(void)
+PyInit_internal(void)
 {
-    return PyModuleDef_Init(&native_internal_module);
+    return PyModuleDef_Init(&librt_internal_module);
 }

mypyc/ir/rtypes.py

@@ -1,9 +1,9 @@
-#ifndef NATIVE_INTERNAL_H
-#define NATIVE_INTERNAL_H
+#ifndef LIBRT_INTERNAL_H
+#define LIBRT_INTERNAL_H
 
-#define NATIVE_INTERNAL_ABI_VERSION 0
+#define LIBRT_INTERNAL_ABI_VERSION 0
 
-#ifdef NATIVE_INTERNAL_MODULE
+#ifdef LIBRT_INTERNAL_MODULE
 
 static PyObject *Buffer_internal(PyObject *source);
 static PyObject *Buffer_internal_empty(void);
@@ -40,17 +40,21 @@ static void **NativeInternal_API;
 #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13])
 
 static int
-import_native_internal(void)
+import_librt_internal(void)
 {
-    NativeInternal_API = (void **)PyCapsule_Import("native_internal._C_API", 0);
+    PyObject *mod = PyImport_ImportModule("librt.internal");
+    if (mod == NULL)
+        return -1;
+    Py_DECREF(mod);  // we import just for the side effect of making the below work.
+    NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0);
     if (NativeInternal_API == NULL)
         return -1;
-    if (NativeInternal_ABI_Version() != NATIVE_INTERNAL_ABI_VERSION) {
-        PyErr_SetString(PyExc_ValueError, "ABI version conflict for native_internal");
+    if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) {
+        PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal");
         return -1;
     }
     return 0;
 }
 
 #endif
-#endif  // NATIVE_INTERNAL_H
+#endif  // LIBRT_INTERNAL_H