Use alternative index url when installing build dependencies

pyodide_build/build_env.py

@@ -16,7 +16,6 @@
 from pyodide_build import __version__
 from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname
 from pyodide_build.config import ConfigManager
-from pyodide_build.recipe import load_all_recipes
 
 RUST_BUILD_PRELUDE = """
 rustup default ${RUST_TOOLCHAIN}
@@ -169,25 +168,6 @@ def get_hostsitepackages() -> str:
     return get_build_flag("HOSTSITEPACKAGES")
 
 
-@functools.cache
-def get_unisolated_packages() -> list[str]:
-    PYODIDE_ROOT = get_pyodide_root()
-
-    unisolated_file = PYODIDE_ROOT / "unisolated.txt"
-    if unisolated_file.exists():
-        # in xbuild env, read from file
-        unisolated_packages = unisolated_file.read_text().splitlines()
-    else:
-        unisolated_packages = []
-        recipe_dir = PYODIDE_ROOT / "packages"
-        recipes = load_all_recipes(recipe_dir)
-        for name, config in recipes.items():
-            if config.build.cross_build_env:
-                unisolated_packages.append(name)
-
-    return unisolated_packages
-
-
 def platform() -> str:
     emscripten_version = get_build_flag("PYODIDE_EMSCRIPTEN_VERSION")
     version = emscripten_version.replace(".", "_")

pyodide_build/common.py

@@ -451,6 +451,41 @@ def to_bool(value: str) -> bool:
     return value.lower() not in {"", "0", "false", "no", "off"}
 
 
+def get_host_platform():
+    """
+    Return a string that identifies the current platform.
+    Simplified version of get_host_platform in pypa/distlib.
+    """
+    if os.name != "posix":
+        raise ValueError(f"only posix platforms are supported, got {os.name}")
+
+    # Set for cross builds explicitly
+    if "_PYTHON_HOST_PLATFORM" in os.environ:
+        return os.environ["_PYTHON_HOST_PLATFORM"]
+
+    # Try to distinguish various flavours of Unix
+    (osname, host, release, version, machine) = os.uname()
+
+    # Convert the OS name to lowercase, remove '/' characters, and translate
+    # spaces (for "Power Macintosh")
+    osname = osname.lower().replace("/", "")
+    machine = machine.replace(" ", "_").replace("/", "-")
+
+    if osname[:5] == "linux":
+        return f"{osname}-{machine}"
+    elif osname[:6] == "darwin":
+        import _osx_support
+        import sysconfig
+
+        osname, release, machine = _osx_support.get_platform_osx(
+            sysconfig.get_config_vars(), osname, release, machine
+        )
+    else:
+        raise ValueError(f"unsupported os: {osname}")
+
+    return f"{osname}-{release}-{machine}"
+
+
 def download_and_unpack_archive(url: str, path: Path, descr: str) -> None:
     """
     Download the cross-build environment from the given URL and extract it to the given path.

pyodide_build/config.py

@@ -179,6 +179,7 @@ def to_env(self) -> dict[str, str]:
     "build_dependency_index_url": "BUILD_DEPENDENCY_INDEX_URL",
     # maintainer only
     "_f2c_fixes_wrapper": "_F2C_FIXES_WRAPPER",
+    "_build_dependency_fallback_to_pypi": "BUILD_DEPENDENCY_FALLBACK_TO_PYPI",
 }
 
 BUILD_VAR_TO_KEY = {v: k for k, v in BUILD_KEY_TO_VAR.items()}
@@ -195,6 +196,7 @@ def to_env(self) -> dict[str, str]:
     "build_dependency_index_url",
     # maintainer only
     "_f2c_fixes_wrapper",
+    "_build_dependency_fallback_to_pypi",
 }
 
 # Default configuration values.
@@ -211,9 +213,10 @@ def to_env(self) -> dict[str, str]:
     # Other configuration
     "pyodide_jobs": "1",
     "skip_emscripten_version_check": "0",
-    "build_dependency_index_url": "https://pypi.anaconda.org/pyodide/simple",
+    "build_dependency_index_url": "https://pypi.anaconda.org/pyodide-build/simple",
     # maintainer only
     "_f2c_fixes_wrapper": "",
+    "_build_dependency_fallback_to_pypi": "1",
 }
 
 # Default configs that are computed from other values (often from Makefile.envs)

pyodide_build/pypabuild.py

@@ -4,9 +4,8 @@
 import subprocess as sp
 import sys
 import traceback
-from collections.abc import Callable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
 from contextlib import contextmanager
-from itertools import chain
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Literal, cast
@@ -18,9 +17,7 @@
 from pyodide_build import _f2c_fixes, common, pywasmcross
 from pyodide_build.build_env import (
     get_build_flag,
-    get_hostsitepackages,
     get_pyversion,
-    get_unisolated_packages,
     platform,
 )
 from pyodide_build.io import _BuildSpecExports
@@ -54,6 +51,8 @@
     "gfortran": "FC",  # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables
 }
 
+HOST_ARCH = common.get_host_platform().replace("-", "_").replace(".", "_")
+
 
 def _gen_runner(
     cross_build_env: Mapping[str, str],
@@ -103,13 +102,6 @@ def symlink_unisolated_packages(env: DefaultIsolatedEnv) -> None:
 
     env_site_packages.mkdir(parents=True, exist_ok=True)
     shutil.copy(sysconfigdata_path, env_site_packages)
-    host_site_packages = Path(get_hostsitepackages())
-    for name in get_unisolated_packages():
-        for path in chain(
-            host_site_packages.glob(f"{name}*"), host_site_packages.glob(f"_{name}*")
-        ):
-            (env_site_packages / path.name).unlink(missing_ok=True)
-            (env_site_packages / path.name).symlink_to(path)
 
 
 def remove_avoided_requirements(
@@ -127,7 +119,7 @@ def install_reqs(env: DefaultIsolatedEnv, reqs: set[str]) -> None:
     env.install(
         remove_avoided_requirements(
             reqs,
-            get_unisolated_packages() + AVOIDED_REQUIREMENTS,
+            AVOIDED_REQUIREMENTS,
         )
     )
 
@@ -153,8 +145,33 @@ def _build_in_isolated_env(
 
         # first install the build dependencies
         symlink_unisolated_packages(env)
-        install_reqs(env, builder.build_system_requires)
-        installed_requires_for_build = False
+        index_url_for_cross_build = get_build_flag("BUILD_DEPENDENCY_INDEX_URL")
+        installed_build_system_requires = (
+            False  # build dependency for in pyproject.toml
+        )
+        installed_backend_requires = (
+            False  # dependencies defined by the backend for a given distribution
+        )
+
+        with switch_index_url(index_url_for_cross_build):
+            try:
+                install_reqs(env, builder.build_system_requires)
+                installed_build_system_requires = True
+            except Exception:
+                pass
+
+        # Disabled for testing
+        if not installed_build_system_requires:
+            if common.to_bool(get_build_flag("BUILD_DEPENDENCY_FALLBACK_TO_PYPI")):
+                print(
+                    f"Failed to install build dependencies from {index_url_for_cross_build}, falling back to default index url"
+                )
+                install_reqs(env, builder.build_system_requires)
+            else:
+                print(
+                    f"Failed to install build dependencies from {index_url_for_cross_build}, proceeding the build, but it will fail."
+                )
+
         try:
             build_reqs = builder.get_requires_for_build(
                 distribution,
@@ -163,10 +180,10 @@ def _build_in_isolated_env(
             pass
         else:
             install_reqs(env, build_reqs)
-            installed_requires_for_build = True
+            installed_backend_requires = True
 
         with common.replace_env(build_env):
-            if not installed_requires_for_build:
+            if not installed_backend_requires:
                 build_reqs = builder.get_requires_for_build(
                     distribution,
                     config_settings,
@@ -242,6 +259,35 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]:
     return env
 
 
+@contextmanager
+def switch_index_url(index_url: str) -> Generator[None, None, None]:
+    """
+    Switch index URL that pip locates the packages.
+    This function is expected to be used during the process of
+    installing package build dependencies.
+
+    Parameters
+    ----------
+    index_url: index URL to switch to
+    """
+
+    env = {
+        "PIP_INDEX_URL": index_url,
+    }
+
+    # For debugging: uncomment the lines below to see the pip error during the package installation
+    # import build
+    # build._ctx.VERBOSITY.set(1)
+
+    # def log(msg, *args, **kwargs):
+    #     print(msg, str(kwargs))
+
+    # build._ctx.LOGGER.set(log)
+
+    with common.replace_env(env) as replaced_env:
+        yield replaced_env
+
+
 @contextmanager
 def get_build_env(
     env: dict[str, str],
@@ -276,6 +322,7 @@ def get_build_env(
         env.update(make_command_wrapper_symlinks(symlink_dir))
 
         sysconfig_dir = Path(get_build_flag("TARGETINSTALLDIR")) / "sysconfigdata"
+        host_pythonpath = Path(get_build_flag("PYTHONPATH"))
         args["PYTHONPATH"] = sys.path + [str(symlink_dir), str(sysconfig_dir)]
         args["orig__name__"] = __name__
         args["pythoninclude"] = get_build_flag("PYTHONINCLUDE")
@@ -290,7 +337,7 @@ def get_build_env(
 
         env["_PYTHON_HOST_PLATFORM"] = platform()
         env["_PYTHON_SYSCONFIGDATA_NAME"] = get_build_flag("SYSCONFIG_NAME")
-        env["PYTHONPATH"] = str(sysconfig_dir)
+        env["PYTHONPATH"] = str(sysconfig_dir) + ":" + str(host_pythonpath)
         env["COMPILER_WRAPPER_DIR"] = str(symlink_dir)
 
         yield env

pyodide_build/tests/conftest.py

@@ -62,7 +62,6 @@ def reset_cache():
     def _reset():
         build_env.get_pyodide_root.cache_clear()
         build_env.get_build_environment_vars.cache_clear()
-        build_env.get_unisolated_packages.cache_clear()
 
     _reset()