python: add a python.build_config option (PEP 739)

Signed-off-by: Filipe Laíns <[email protected]>

Author: FFY00

docs/markdown/Builtin-options.md

@@ -408,6 +408,7 @@ install prefix. For example: if the install prefix is `/usr` and the
 | platlibdir        |               | Directory path              | Directory for site-specific, platform-specific files (Since 0.60.0) |
 | purelibdir        |               | Directory path              | Directory for site-specific, non-platform-specific files  (Since 0.60.0) |
 | allow_limited_api | true          | true, false                 | Disables project-wide use of the Python Limited API (Since 1.3.0) |
+| build_config      |               | File path                   | Specifies the Python build configuration file (PEP 739) (Since 1.7.0) |
 
 *Since 0.60.0* The `python.platlibdir` and `python.purelibdir` options are used
 by the python module methods `python.install_sources()` and

mesonbuild/dependencies/python.py

@@ -76,7 +76,8 @@ def __init__(self, name: str, environment: Environment, kwargs: T.Dict[str, T.An
 
 class BasicPythonExternalProgram(ExternalProgram):
     def __init__(self, name: str, command: T.Optional[T.List[str]] = None,
-                 ext_prog: T.Optional[ExternalProgram] = None):
+                 ext_prog: T.Optional[ExternalProgram] = None,
+                 build_config_path: T.Optional[str] = None):
         if ext_prog is None:
             super().__init__(name, command=command, silent=True)
         else:
@@ -86,6 +87,15 @@ def __init__(self, name: str, command: T.Optional[T.List[str]] = None,
             self.cached_version = None
             self.version_arg = '--version'
 
+        self.build_config = None
+
+        if build_config_path:
+            try:
+                with open(build_config_path, encoding='utf8') as f:
+                    self.build_config = json.load(f)
+            except OSError as e:
+                mlog.warning(f'Failed to read python.build_config: {e}')
+
         # We want strong key values, so we always populate this with bogus data.
         # Otherwise to make the type checkers happy we'd have to do .get() for
         # everycall, even though we know that the introspection data will be
@@ -116,6 +126,14 @@ def _check_version(self, version: str) -> bool:
     def sanity(self) -> bool:
         # Sanity check, we expect to have something that at least quacks in tune
 
+        if self.build_config:
+            if not self.build_config['libpython']:
+                mlog.debug('This Python installation does not provide a libpython')
+                return False
+            if not self.build_config['c_api']:
+                mlog.debug('This Python installation does support the C API')
+                return False
+
         import importlib.resources
 
         with importlib.resources.path('mesonbuild.scripts', 'python_info.py') as f:
@@ -145,12 +163,24 @@ class _PythonDependencyBase(_Base):
 
     def __init__(self, python_holder: 'BasicPythonExternalProgram', embed: bool):
         self.embed = embed
-        self.version: str = python_holder.info['version']
-        self.platform = python_holder.info['platform']
-        self.variables = python_holder.info['variables']
+        self.build_config = python_holder.build_config
+
+        if self.build_config:
+            self.version = self.build_config['language']['version']
+            self.platform = self.build_config['platform']
+            self.is_freethreaded = 't' in self.build_config['abi']['flags']
+            self.link_libpython = self.build_config['libpython']['link_extensions']
+        else:
+            self.version = python_holder.info['version']
+            self.platform = python_holder.info['platform']
+            self.is_freethreaded = python_holder.info['is_freethreaded']
+            self.link_libpython = python_holder.info['link_libpython']
+            # This data shouldn't be needed when build_config is set
+            self.is_pypy = python_holder.info['is_pypy']
+            self.variables = python_holder.info['variables']
+
         self.paths = python_holder.info['paths']
-        self.is_pypy = python_holder.info['is_pypy']
-        self.is_freethreaded = python_holder.info['is_freethreaded']
+
         # The "-embed" version of python.pc / python-config was introduced in 3.8,
         # and distutils extension linking was changed to be considered a non embed
         # usage. Before then, this dependency always uses the embed=True handling
@@ -159,7 +189,9 @@ def __init__(self, python_holder: 'BasicPythonExternalProgram', embed: bool):
         # On macOS and some Linux distros (Debian) distutils doesn't link extensions
         # against libpython, even on 3.7 and below. We call into distutils and
         # mirror its behavior. See https://github.com/mesonbuild/meson/issues/4117
-        self.link_libpython = python_holder.info['link_libpython'] or embed
+        if not self.link_libpython:
+            self.link_libpython = embed
+
         self.info: T.Optional[T.Dict[str, str]] = None
         if mesonlib.version_compare(self.version, '>= 3.0'):
             self.major_version = 3
@@ -173,20 +205,27 @@ def __init__(self, python_holder: 'BasicPythonExternalProgram', embed: bool):
             self.compile_args += ['-DPy_GIL_DISABLED']
 
     def find_libpy(self, environment: 'Environment') -> None:
-        if self.is_pypy:
-            if self.major_version == 3:
-                libname = 'pypy3-c'
-            else:
-                libname = 'pypy-c'
-            libdir = os.path.join(self.variables.get('base'), 'bin')
-            libdirs = [libdir]
+        if self.build_config:
+            path = self.build_config['libpython'].get('dynamic')
+            if not path:
+                raise DependencyException('Python does not provide a dynamic libpython library')
+            libdirs = [os.path.dirname(path)]
+            libname = os.path.basename(path)
         else:
-            libname = f'python{self.version}'
-            if 'DEBUG_EXT' in self.variables:
-                libname += self.variables['DEBUG_EXT']
-            if 'ABIFLAGS' in self.variables:
-                libname += self.variables['ABIFLAGS']
-            libdirs = []
+            if self.is_pypy:
+                if self.major_version == 3:
+                    libname = 'pypy3-c'
+                else:
+                    libname = 'pypy-c'
+                libdir = os.path.join(self.variables.get('base'), 'bin')
+                libdirs = [libdir]
+            else:
+                libname = f'python{self.version}'
+                if 'DEBUG_EXT' in self.variables:
+                    libname += self.variables['DEBUG_EXT']
+                if 'ABIFLAGS' in self.variables:
+                    libname += self.variables['ABIFLAGS']
+                libdirs = []
 
         largs = self.clib_compiler.find_library(libname, environment, libdirs)
         if largs is not None:
@@ -212,6 +251,15 @@ def get_windows_python_arch(self) -> str:
         raise DependencyException('Unknown Windows Python platform {self.platform!r}')
 
     def get_windows_link_args(self, limited_api: bool) -> T.Optional[T.List[str]]:
+        if self.build_config:
+            if self.static:
+                key = 'static'
+            elif limited_api:
+                key = 'dynamic-stableabi'
+            else:
+                key = 'dynamic'
+            return [self.build_config['libpython'][key]]
+
         if self.platform.startswith('win'):
             vernum = self.variables.get('py_version_nodot')
             verdot = self.variables.get('py_version_short')
@@ -354,10 +402,13 @@ def __init__(self, name: str, environment: 'Environment',
             self.is_found = True
 
         # compile args
-        inc_paths = mesonlib.OrderedSet([
-            self.variables.get('INCLUDEPY'),
-            self.paths.get('include'),
-            self.paths.get('platinclude')])
+        if self.build_config:
+            inc_paths = mesonlib.OrderedSet([self.build_config['c_api']['headers']])
+        else:
+            inc_paths = mesonlib.OrderedSet([
+                self.variables.get('INCLUDEPY'),
+                self.paths.get('include'),
+                self.paths.get('platinclude')])
 
         self.compile_args += ['-I' + path for path in inc_paths if path]
 
@@ -386,11 +437,18 @@ def python_factory(env: 'Environment', for_machine: 'MachineChoice',
     if installation is None:
         installation = BasicPythonExternalProgram('python3', mesonlib.python_command)
         installation.sanity()
-    pkg_version = installation.info['variables'].get('LDVERSION') or installation.info['version']
+
+    if installation.build_config:
+        pkg_version = installation.build_config['language']['version']
+    else:
+        pkg_version = installation.info['variables'].get('LDVERSION') or installation.info['version']
 
     if DependencyMethods.PKGCONFIG in methods:
         if from_installation:
-            pkg_libdir = installation.info['variables'].get('LIBPC')
+            if installation.build_config:
+                pkg_libdir = installation.build_config['c_api']['pkgconfig_path']
+            else:
+                pkg_libdir = installation.info['variables'].get('LIBPC')
             pkg_embed = '-embed' if embed and mesonlib.version_compare(installation.info['version'], '>=3.8') else ''
             pkg_name = f'python-{pkg_version}{pkg_embed}'
 

mesonbuild/modules/python.py

@@ -115,17 +115,27 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'):
         info = python.info
         prefix = self.interpreter.environment.coredata.get_option(OptionKey('prefix'))
         assert isinstance(prefix, str), 'for mypy'
+
+        if python.build_config:
+            self.version = python.build_config['language']['version']
+            self.platform = python.build_config['platform']
+            self.suffix = python.build_config['abi']['extension_suffix']
+            self.limited_api_suffix = python.build_config['abi']['stable_abi_suffix']
+            self.link_libpython = python.build_config['libpython']['link_extensions']
+            self.is_pypy = python.build_config['implementation']['name'] == 'pypy'
+        else:
+            self.version = info['version']
+            self.platform = info['platform']
+            self.suffix = info['suffix']
+            self.limited_api_suffix = info['limited_api_suffix']
+            self.link_libpython = info['link_libpython']
+            self.is_pypy = info['is_pypy']
+
         self.variables = info['variables']
-        self.suffix = info['suffix']
-        self.limited_api_suffix = info['limited_api_suffix']
         self.paths = info['paths']
         self.pure = python.pure
         self.platlib_install_path = os.path.join(prefix, python.platlib)
         self.purelib_install_path = os.path.join(prefix, python.purelib)
-        self.version = info['version']
-        self.platform = info['platform']
-        self.is_pypy = info['is_pypy']
-        self.link_libpython = info['link_libpython']
         self.methods.update({
             'extension_module': self.extension_module_method,
             'dependency': self.dependency_method,
@@ -256,8 +266,12 @@ def _dependency_method_impl(self, kwargs: TYPE_kwargs) -> Dependency:
         if dep is not None:
             return dep
 
+        build_config = self.interpreter.environment.coredata.get_option(OptionKey('python.build_config'))
+
         new_kwargs = kwargs.copy()
         new_kwargs['required'] = False
+        if build_config:
+            new_kwargs['build_config'] = build_config
         candidates = python_factory(self.interpreter.environment, for_machine, new_kwargs, self.held_object)
         dep = find_external_dependency('python', self.interpreter.environment, new_kwargs, candidates)
 
@@ -441,11 +455,13 @@ def _get_win_pythonpath(name_or_path: str) -> T.Optional[str]:
             return None
 
     def _find_installation_impl(self, state: 'ModuleState', display_name: str, name_or_path: str, required: bool) -> MaybePythonProg:
+        build_config = self.interpreter.environment.coredata.get_option(OptionKey('python.build_config'))
+
         if not name_or_path:
-            python = PythonExternalProgram('python3', mesonlib.python_command)
+            python = PythonExternalProgram('python3', mesonlib.python_command, build_config_path=build_config)
         else:
             tmp_python = ExternalProgram.from_entry(display_name, name_or_path)
-            python = PythonExternalProgram(display_name, ext_prog=tmp_python)
+            python = PythonExternalProgram(display_name, ext_prog=tmp_python, build_config_path=build_config)
 
             if not python.found() and mesonlib.is_windows():
                 pythonpath = self._get_win_pythonpath(name_or_path)
@@ -459,7 +475,7 @@ def _find_installation_impl(self, state: 'ModuleState', display_name: str, name_
             # it
             if not python.found() and name_or_path in {'python2', 'python3'}:
                 tmp_python = ExternalProgram.from_entry(display_name, 'python')
-                python = PythonExternalProgram(name_or_path, ext_prog=tmp_python)
+                python = PythonExternalProgram(name_or_path, ext_prog=tmp_python, build_config_path=build_config)
 
         if python.found():
             if python.sanity(state):

mesonbuild/options.py

@@ -663,6 +663,8 @@ def add_to_argparse(self, name: str, parser: argparse.ArgumentParser, help_suffi
      BuiltinOption(UserStringOption, 'Directory for site-specific, non-platform-specific files.', '')),
     (OptionKey('python.allow_limited_api'),
      BuiltinOption(UserBooleanOption, 'Whether to allow use of the Python Limited API', True)),
+    (OptionKey('python.build_config'),
+     BuiltinOption(UserStringOption, 'Config file containing the build details for the target Python installation.', '')),
 ])
 
 BUILTIN_OPTIONS = OrderedDict(chain(BUILTIN_DIR_OPTIONS.items(), BUILTIN_CORE_OPTIONS.items()))