Bug 1886479 - Introduce the possibility to provide a fallback in python site r=mach-reviewers,ahal

pyyaml is already vendored but as a consequence we don't benefit from
the faster native loader.

Introduce the possibility to provide a vendored package as fallback to
an optional pypi package. If the pypi package install fails, we can
still use the vendored one without the speed benefit.

This makes my `mach build export` step 8% faster.

Differential Revision: https://phabricator.services.mozilla.com/D234652

Author: serge-sans-paille

python/docs/index.rst

@@ -160,6 +160,18 @@ By default ``./mach vendor python`` only fully runs if changes are detected in t
 want to force the full vendor to run, just add ``--force``.
 
 
+If the package contains optional native dependencies, they won't be compiled as
+part of the vendored package. It is however possible to prefer the pypi version
+which may contain the native bits, while allowing to fallback to the vendored
+version:
+
+.. code:: text
+
+    ...
+    vendored-fallback:pypi-package-name:third_party/python/new-package:explanation
+    ...
+
+
 .. _mach-and-build-native-dependencies:
 
 Mach/Build Native 3rd-party Dependencies

python/mach/mach/requirements.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
+import re
 from pathlib import Path
 
 from packaging.requirements import Requirement
@@ -63,13 +64,20 @@ def __init__(self):
         self.pypi_requirements = []
         self.pypi_optional_requirements = []
         self.vendored_requirements = []
+        self.vendored_fallback_requirements = []
 
     def pths_as_absolute(self, topsrcdir: str):
         return [
             os.path.normcase(Path(topsrcdir) / pth.path)
             for pth in (self.pth_requirements + self.vendored_requirements)
         ]
 
+    def pths_fallback_as_absolute(self, topsrcdir: str):
+        return [
+            os.path.normcase(Path(topsrcdir) / pth.path)
+            for pth in self.vendored_fallback_requirements
+        ]
+
     @classmethod
     def from_requirements_definition(
         cls,
@@ -124,6 +132,32 @@ def _parse_requirements_line(
             requirements_output.pth_requirements.append(PthSpecifier(params))
         elif action == "vendored":
             requirements_output.vendored_requirements.append(PthSpecifier(params))
+        elif action == "vendored-fallback":
+            if is_thunderbird_packages_txt:
+                raise Exception(THUNDERBIRD_PYPI_ERROR)
+
+            pypi_pkg, vendored_path, repercussion = params.split(":")
+            requirements = topsrcdir / "third_party" / "python" / "requirements.txt"
+            with open(requirements) as req:
+                content = req.read()
+                pattern = re.compile(rf"^({pypi_pkg}==.*) \\$", re.MULTILINE)
+                version_matches = pattern.findall(content, re.MULTILINE)
+                if len(version_matches) != 1:
+                    raise Exception(
+                        f"vendored-fallback package {pypi_pkg} is not referenced in {requirements}"
+                    )
+                (raw_requirement,) = version_matches
+
+            requirements_output.pypi_optional_requirements.append(
+                PypiOptionalSpecifier(
+                    repercussion,
+                    _parse_package_specifier(raw_requirement, only_strict_requirements),
+                )
+            )
+
+            requirements_output.vendored_fallback_requirements.append(
+                PthSpecifier(vendored_path)
+            )
         elif action == "packages.txt":
             _parse_requirements_definition_file(
                 topsrcdir / params,

python/mach/mach/site.py

@@ -412,6 +412,7 @@ def _sys_path(self):
                 *stdlib_paths,
                 *self._requirements.pths_as_absolute(self._topsrcdir),
                 *system_site_paths,
+                *self._requirements.pths_fallback_as_absolute(self._topsrcdir),
             ]
         elif self._site_packages_source == SitePackagesSource.NONE:
             stdlib_paths = self._metadata.original_python.sys_path_stdlib()
@@ -795,6 +796,12 @@ def _pthfile_lines(self):
         lines.extend(
             _deprioritize_venv_packages(self._virtualenv, self._populate_virtualenv)
         )
+        # Except for fallback that should come after the pip installed package.
+        lines.extend(
+            resolve_requirements(self._topsrcdir, "mach").pths_fallback_as_absolute(
+                self._topsrcdir
+            )
+        )
 
         # Note that an on-disk virtualenv is always created for commands, even if they
         # are using the system as their site-packages source. This is to support use

python/sites/mach.txt

@@ -57,7 +57,7 @@ pth:xpcom/geckoprocesstypes_generator
 pth:xpcom/idl-parser
 vendored:third_party/python/Jinja2
 vendored:third_party/python/MarkupSafe/src
-vendored:third_party/python/PyYAML/lib/
+vendored-fallback:pyyaml:third_party/python/PyYAML/lib/:faster native loading is disabled
 vendored:third_party/python/ansicon
 vendored:third_party/python/appdirs
 vendored:third_party/python/attrs