ENH: fix support for Python limited API / stable ABI wheels

Meson gained support for building Python extension modules targeting
the Python limited API, therefore it is time for meson-python to
properly support tagging the build wheels as targeting the stable ABI.

Unfortunately there isn't a reliable and cross-platform way to detect
when extension modules are build for the limited API. Therefore, we
need to add an explicit "limited-api" configuration option in the
[tool.meson-python] section in pyproject.toml.

Author: dnicolodi

docs/reference/pyproject-settings.rst

@@ -13,6 +13,19 @@ This page lists the configuration settings supported by
 :ref:`how-to-guides-meson-args` guide for for information on how to
 use them and examples.
 
+.. option:: tool.meson-python.limited-api
+
+   A boolean indicating whether the extension modules contained in the
+   Python package target the `Python limited API`__.  Extension
+   modules can be compiled for the Python limited API specifying the
+   ``limited_api`` argument to the |extension_module()|__ function
+   in the Meson Python module.  When this setting is set to true, the
+   value ``abi3`` is used for the Python wheel filename ABI tag.
+
+   This setting is automatically reverted to false when the
+   ``-Dpython.allow_limited_api=false`` option is passed to ``meson
+   setup``.
+
 .. option:: tool.meson-python.args.dist
 
    Extra arguments to be passed to the ``meson dist`` command.
@@ -28,3 +41,9 @@ use them and examples.
 .. option:: tool.meson-python.args.install
 
    Extra arguments to be passed to the ``meson install`` command.
+
+
+__ https://docs.python.org/3/c-api/stable.html?highlight=limited%20api#stable-application-binary-interface
+__ https://mesonbuild.com/Python-module.html#extension_module
+
+.. |extension_module()| replace:: ``extension_module()``

mesonpy/__init__.py

@@ -128,9 +128,8 @@ def _init_colors() -> Dict[str, str]:
 
 
 _SUFFIXES = importlib.machinery.all_suffixes()
-_EXTENSION_SUFFIXES = importlib.machinery.EXTENSION_SUFFIXES
-_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P<abi>[^.]+)\.)?(?:so|pyd|dll)$')
-assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES)
+_EXTENSION_SUFFIX_REGEX = re.compile(r'^[^.]+\.(?:(?P<abi>[^.]+)\.)?(?:so|pyd|dll)$')
+assert all(re.match(_EXTENSION_SUFFIX_REGEX, f'foo{x}') for x in importlib.machinery.EXTENSION_SUFFIXES)
 
 
 # Map Meson installation path placeholders to wheel installation paths.
@@ -344,34 +343,20 @@ def entrypoints_txt(self) -> bytes:
 
     @cached_property
     def _stable_abi(self) -> Optional[str]:
-        """Determine stabe ABI compatibility.
-
-        Examine all files installed in {platlib} that look like
-        extension modules (extension .pyd on Windows, .dll on Cygwin,
-        and .so on other platforms) and, if they all share the same
-        PEP 3149 filename stable ABI tag, return it.
-
-        Other files are ignored.
-
-        """
-        soext = sorted(_EXTENSION_SUFFIXES, key=len)[0]
-        abis = []
-
-        for path, _ in self._wheel_files['platlib']:
-            # NOTE: When searching for shared objects files, we assume the host
-            # and build machines have the same soext, even though that we might
-            # be cross compiling.
-            if path.suffix == soext:
-                match = re.match(r'^[^.]+(.*)$', path.name)
-                assert match is not None
-                suffix = match.group(1)
-                match = _EXTENSION_SUFFIX_REGEX.match(suffix)
+        if self._project._limited_api:
+            # Verify stabe ABI compatibility: examine files installed
+            # in {platlib} that look like extension modules, and raise
+            # an exception if any of them has a Python version
+            # specific extension filename suffix ABI tag.
+            for path, _ in self._wheel_files['platlib']:
+                match = _EXTENSION_SUFFIX_REGEX.match(path.name)
                 if match:
-                    abis.append(match.group('abi'))
-
-        stable = [x for x in abis if x and re.match(r'abi\d+', x)]
-        if len(stable) > 0 and len(stable) == len(abis) and all(x == stable[0] for x in stable[1:]):
-            return stable[0]
+                    abi = match.group('abi')
+                    if abi is not None and abi != 'abi3':
+                        raise BuildError(
+                            f'The package declares compatibility with Python limited API but extension '
+                            f'module {os.fspath(path)!r} is tagged for a specific Python version.')
+            return 'abi3'
         return None
 
     @property
@@ -576,10 +561,16 @@ def _strings(value: Any, name: str) -> List[str]:
             raise ConfigError(f'Configuration entry "{name}" must be a list of strings')
         return value
 
+    def _bool(value: Any, name: str) -> bool:
+        if not isinstance(value, bool):
+            raise ConfigError(f'Configuration entry "{name}" must be a boolean')
+        return value
+
     scheme = _table({
+        'limited-api': _bool,
         'args': _table({
             name: _strings for name in _MESON_ARGS_KEYS
-        })
+        }),
     })
 
     table = pyproject.get('tool', {}).get('meson-python', {})
@@ -632,7 +623,7 @@ class Project():
     ]
     _metadata: pyproject_metadata.StandardMetadata
 
-    def __init__(
+    def __init__(  # noqa: C901
         self,
         source_dir: Path,
         working_dir: Path,
@@ -648,6 +639,7 @@ def __init__(
         self._meson_native_file = self._build_dir / 'meson-python-native-file.ini'
         self._meson_cross_file = self._build_dir / 'meson-python-cross-file.ini'
         self._meson_args: MesonArgs = collections.defaultdict(list)
+        self._limited_api = False
 
         _check_meson_version()
 
@@ -730,6 +722,15 @@ def __init__(
         if 'version' in self._metadata.dynamic:
             self._metadata.version = packaging.version.Version(self._meson_version)
 
+        # limited API
+        self._limited_api = pyproject_config.get('limited-api', False)
+        if self._limited_api:
+            # check whether limited API is disabled for the Meson project
+            options = self._info('intro-buildoptions')
+            value = next((option['value'] for option in options if option['name'] == 'python.allow_limited_api'), None)
+            if not value:
+                self._limited_api = False
+
     def _run(self, cmd: Sequence[str]) -> None:
         """Invoke a subprocess."""
         # Flush the line to ensure that the log line with the executed

tests/packages/limited-api/meson.build

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2023 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+project('limited-api', 'c', version: '1.0.0')
+
+py = import('python').find_installation(pure: false)
+
+py.extension_module(
+  'module',
+  'module.c',
+  limited_api: '3.7',
+  install: true,
+)
+
+if get_option('extra')
+  py.extension_module(
+    'extra',
+    'module.c',
+    install: true,
+  )
+endif

tests/packages/limited-api/meson.options

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2023 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+option('extra', type: 'boolean', value: false)

tests/packages/limited-api/module.c

@@ -0,0 +1,31 @@
+// SPDX-FileCopyrightText: 2023 The meson-python developers
+//
+// SPDX-License-Identifier: MIT
+
+#include <Python.h>
+
+static PyObject* add(PyObject *self, PyObject *args) {
+    int a, b;
+
+    if (!PyArg_ParseTuple(args, "ii", &a, &b))
+        return NULL;
+
+    return PyLong_FromLong(a + b);
+}
+
+static PyMethodDef methods[] = {
+    {"add", add, METH_VARARGS, NULL},
+    {NULL, NULL, 0, NULL},
+};
+
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT,
+    "plat",
+    NULL,
+    -1,
+    methods,
+};
+
+PyMODINIT_FUNC PyInit_module(void) {
+    return PyModule_Create(&module);
+}

tests/packages/limited-api/pyproject.toml

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+[build-system]
+build-backend = 'mesonpy'
+requires = ['meson-python']
+
+[tool.meson-python]
+limited-api = true

tests/test_tags.py

@@ -2,9 +2,11 @@
 #
 # SPDX-License-Identifier: MIT
 
+import importlib.machinery
 import os
 import pathlib
 import platform
+import sys
 import sysconfig
 
 from collections import defaultdict
@@ -24,8 +26,17 @@
 INTERPRETER = tag.interpreter
 PLATFORM = adjust_packaging_platform_tag(tag.platform)
 
+
+def get_abi3_suffix():
+    for suffix in importlib.machinery.EXTENSION_SUFFIXES:
+        if '.abi3' in suffix:  # Unix
+            return suffix
+        elif suffix == '.pyd':  # Windows
+            return suffix
+
+
 SUFFIX = sysconfig.get_config_var('EXT_SUFFIX')
-ABI3SUFFIX = next((x for x in mesonpy._EXTENSION_SUFFIXES if '.abi3.' in x), None)
+ABI3SUFFIX = get_abi3_suffix()
 
 
 def test_wheel_tag():
@@ -52,11 +63,13 @@ def test_python_host_platform(monkeypatch):
     assert mesonpy._tags.get_platform_tag().endswith('x86_64')
 
 
-def wheel_builder_test_factory(monkeypatch, content):
+def wheel_builder_test_factory(monkeypatch, content, limited_api=False):
     files = defaultdict(list)
     files.update({key: [(pathlib.Path(x), os.path.join('build', x)) for x in value] for key, value in content.items()})
     monkeypatch.setattr(mesonpy._WheelBuilder, '_wheel_files', files)
-    return mesonpy._WheelBuilder(None, None, pathlib.Path(), pathlib.Path(), pathlib.Path())
+    class Project:
+        _limited_api = limited_api
+    return mesonpy._WheelBuilder(Project(), None, pathlib.Path(), pathlib.Path(), pathlib.Path())
 
 
 def test_tag_empty_wheel(monkeypatch):
@@ -78,17 +91,18 @@ def test_tag_platlib_wheel(monkeypatch):
     assert str(builder.tag) == f'{INTERPRETER}-{ABI}-{PLATFORM}'
 
 
-@pytest.mark.skipif(not ABI3SUFFIX, reason='Stable ABI not supported by Python interpreter')
 def test_tag_stable_abi(monkeypatch):
     builder = wheel_builder_test_factory(monkeypatch, {
         'platlib': [f'extension{ABI3SUFFIX}'],
-    })
+    }, True)
     assert str(builder.tag) == f'{INTERPRETER}-abi3-{PLATFORM}'
 
 
-@pytest.mark.skipif(not ABI3SUFFIX, reason='Stable ABI not supported by Python interpreter')
+@pytest.mark.skipif(sys.version_info < (3, 8) and platform.system() == 'Windows',
+                    reason='Extension modules filename suffix without ABI tags')
 def test_tag_mixed_abi(monkeypatch):
     builder = wheel_builder_test_factory(monkeypatch, {
         'platlib': [f'extension{ABI3SUFFIX}', f'another{SUFFIX}'],
-    })
-    assert str(builder.tag) == f'{INTERPRETER}-{ABI}-{PLATFORM}'
+    }, True)
+    with pytest.raises(mesonpy.BuildError, match='The package declares compatibility with Python limited API but '):
+        assert str(builder.tag) == f'{INTERPRETER}-abi3-{PLATFORM}'

tests/test_wheel.py

@@ -282,3 +282,32 @@ def test_skip_subprojects(package_subproject, tmp_path, arg):
         'subproject-1.0.0.dist-info/WHEEL',
         'subproject.py',
     }
+
+
+# Requires Meson 1.3.0, see https://github.com/mesonbuild/meson/pull/11745.
+@pytest.mark.skipif(MESON_VERSION < (1, 2, 99), reason='Meson version too old')
+def test_limited_api(wheel_limited_api):
+    artifact = wheel.wheelfile.WheelFile(wheel_limited_api)
+    name = artifact.parsed_filename
+    assert name.group('pyver') == INTERPRETER
+    assert name.group('abi') == 'abi3'
+    assert name.group('plat') == PLATFORM
+
+
+# Requires Meson 1.3.0, see https://github.com/mesonbuild/meson/pull/11745.
+@pytest.mark.skipif(MESON_VERSION < (1, 2, 99), reason='Meson version too old')
+def test_limited_api_bad(package_limited_api, tmp_path):
+    with pytest.raises(mesonpy.BuildError, match='The package declares compatibility with Python limited API but '):
+        with mesonpy.Project.with_temp_working_dir(meson_args={'setup': ['-Dextra=true']}) as project:
+            project.wheel(tmp_path)
+
+
+# Requires Meson 1.3.0, see https://github.com/mesonbuild/meson/pull/11745.
+@pytest.mark.skipif(MESON_VERSION < (1, 2, 99), reason='Meson version too old')
+def test_limited_api_disabled(package_limited_api, tmp_path):
+    filename = mesonpy.build_wheel(tmp_path, {'setup-args': ['-Dpython.allow_limited_api=false']})
+    artifact = wheel.wheelfile.WheelFile(tmp_path / filename)
+    name = artifact.parsed_filename
+    assert name.group('pyver') == INTERPRETER
+    assert name.group('abi') == ABI
+    assert name.group('plat') == PLATFORM