Add support for constraints, install in topological order

CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Changed
+
+- `micropip.install` now installs packages in topological order: previously all packages
+  from `pyodide-lock.json` were installed before any wheels from indexes or custom URLs.
+  [#177](https://github.com/pyodide/micropip/pull/177)
+
+### Added
+
+- Added support for constraining resolved requirements via
+  `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])`.
+  [#177](https://github.com/pyodide/micropip/pull/177)
+
 ### Fixed
 
 - Fix a bug that prevented non-standard relative urls to be treated as such

docs/project/usage.md

@@ -71,18 +71,79 @@ You can pass multiple packages to `micropip.install`:
 await micropip.install(["pkg1", "pkg2"])
 ```
 
-You can specify additional constraints:
+A dependency can be refined as per the [PEP-508] spec:
+
+[pep-508]: https://peps.python.org/pep-0508
 
 ```python
 await micropip.install("snowballstemmer==2.2.0")
 await micropip.install("snowballstemmer>=2.2.0")
+await micropip.install("snowballstemmer @ https://.../snowballstemmer.*.whl")
 await micropip.install("snowballstemmer[all]")
 ```
 
+### Disabling dependency resolution
+
 micropip does dependency resolution by default, but you can disable it,
 this is useful if you want to install a package that has a dependency
 which is not a pure Python package, but it is not mandatory for your use case:
 
 ```python
 await micropip.install("pkg", deps=False)
 ```
+
+### Constraining indirect dependencies
+
+Dependency resolution can be further customized with optional `constraints`:
+these modify both _direct_ and _indirect_ dependency resolutions, while direct URLs
+in either a requirement or constraint will bypass any other specifiers.
+
+As described in the [`pip` documentation][pip-constraints], each constraint:
+
+[pip-constraints]: https://pip.pypa.io/en/stable/user_guide/#constraints-files
+
+  - _must_ provide a name
+  - _must_ provide exactly one of
+    - a set of version specifiers
+    - a URL
+  - _must not_  request any `[extras]`
+
+Multiple constraints of the same canonical name are merged.
+
+Invalid constraints will be silently discarded, or logged if `verbose` is provided.
+
+```python
+await micropip.install(
+    "pkg",
+    constraints=[
+        "other-pkg==0.1.1",
+        "some-other-pkg<2",
+        "some-other-pkg<3",                           # merged with the above
+        "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl",
+        # silently discarded                          # why?
+        "yet-another-pkg >=1",                        # previously defined by URL
+        "yet_another_pkg-0.1.2-py3-none-any.whl",     # missing name
+        "something-completely[different] ==0.1.1",    # extras
+        "package-with-no-version",                    # missing version or URL
+        "other-pkg ==0.0.1 ; python_version < '3'",   # not applicable
+    ]
+)
+```
+
+Over-constrained requirements will fail to resolve, leaving the environment unmodified.
+
+```python
+await micropip.install("pkg ==1", constraints=["pkg ==2"])
+# ValueError: Can't find a pure Python 3 wheel for 'pkg==1,==2'.
+```
+
+### Setting default constraints
+
+`micropip.set_constraints` replaces any default constraints for all subsequent
+calls to `micropip.install` that don't specify `constraints`:
+
+```python
+micropip.set_constraints(["other-pkg ==0.1.1"])
+await micropip.install("pkg")                         # uses defaults, if needed
+await micropip.install("another-pkg", constraints=[]) # ignores defaults
+```

micropip/__init__.py

@@ -9,6 +9,7 @@
 
 install = _package_manager_singleton.install
 set_index_urls = _package_manager_singleton.set_index_urls
+set_constraints = _package_manager_singleton.set_constraints
 list = _package_manager_singleton.list
 freeze = _package_manager_singleton.freeze
 add_mock_package = _package_manager_singleton.add_mock_package

micropip/_utils.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from sysconfig import get_config_var, get_platform
 
-from packaging.requirements import Requirement
+from packaging.requirements import InvalidRequirement, Requirement
 from packaging.tags import Tag
 from packaging.tags import sys_tags as sys_tags_orig
 from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name
@@ -268,3 +268,92 @@ def fix_package_dependencies(
     (get_dist_info(dist) / "PYODIDE_REQUIRES").write_text(
         json.dumps(sorted(x for x in depends))
     )
+
+
+def validate_constraints(
+    constraints: list[str] | None,
+    environment: dict[str, str] | None = None,
+) -> tuple[dict[str, Requirement], dict[str, list[str]]]:
+    """Build a validated ``Requirement`` dictionary from raw constraint strings.
+
+    Parameters
+    ----------
+    constraints (list):
+        A list of PEP-508 dependency specs, expected to contain both a package
+        name and at least one specifier or a direct URL.
+
+    environment (optional dict):
+        The markers for the current environment, such as OS, Python implementation.
+        If ``None``, the current execution environment will be used.
+
+    Returns
+    -------
+        A 2-tuple of:
+        - a dictionary of ``Requirement`` objects, keyed by canonical name
+        - a dictionary of message strings, keyed by constraint
+    """
+    constrained_reqs: dict[str, Requirement] = {}
+    ignore_messages: dict[str, list[str]] = {}
+
+    for raw_constraint in constraints or []:
+        constraint_messages: list[str] = []
+
+        try:
+            req = Requirement(raw_constraint)
+            req.name = canonicalize_name(req.name)
+        except InvalidRequirement as err:
+            ignore_messages[raw_constraint] = [f"failed to parse: {err}"]
+            continue
+
+        if req.extras:
+            constraint_messages.append("may not provide [extras]")
+
+        if not (req.url or len(req.specifier)):
+            constraint_messages.append("no version or URL")
+
+        if req.marker and not req.marker.evaluate(environment):
+            constraint_messages.append(f"not applicable: {req.marker}")
+
+        if constraint_messages:
+            ignore_messages[raw_constraint] = constraint_messages
+        elif req.name in constrained_reqs:
+            ignore_messages[raw_constraint] = [
+                f"updated existing constraint for {req.name}"
+            ]
+            constrained_reqs[req.name] = constrain_requirement(req, constrained_reqs)
+        else:
+            constrained_reqs[req.name] = req
+
+    return constrained_reqs, ignore_messages
+
+
+def constrain_requirement(
+    requirement: Requirement, constrained_requirements: dict[str, Requirement]
+) -> Requirement:
+    """Refine or replace a requirement from a set of constraints.
+
+    Parameters
+    ----------
+    requirement (list):
+        A list of PEP-508 dependency specs, expected to contain both a package
+        name and at least one speicifier.
+
+    Returns
+    -------
+        A 2-tuple of:
+        - a dictionary of ``Requirement`` objects, keyed by canonical name
+        - a dictionary of messages strings, keyed by constraint
+    """
+    # URLs cannot be merged
+    if requirement.url:
+        return requirement
+
+    as_constrained = constrained_requirements.get(canonicalize_name(requirement.name))
+
+    if as_constrained:
+        if as_constrained.url:
+            requirement = as_constrained
+        else:
+            requirement.specifier = requirement.specifier & as_constrained.specifier
+
+    return requirement

micropip/install.py

@@ -1,15 +1,16 @@
 import asyncio
 import importlib
-from collections.abc import Coroutine
+from graphlib import CycleError, TopologicalSorter
 from pathlib import Path
-from typing import Any
 
 from packaging.markers import default_environment
 
 from ._compat import loadPackage, to_js
 from .constants import FAQ_URLS
 from .logging import setup_logging
+from .package import PackageMetadata
 from .transaction import Transaction
+from .wheelinfo import WheelInfo
 
 
 async def install(
@@ -20,6 +21,7 @@ async def install(
     credentials: str | None = None,
     pre: bool = False,
     *,
+    constraints: list[str] | None = None,
     verbose: bool | int | None = None,
 ) -> None:
     with setup_logging().ctx_level(verbose) as logger:
@@ -49,6 +51,7 @@ async def install(
             fetch_kwargs=fetch_kwargs,
             verbose=verbose,
             index_urls=index_urls,
+            constraints=constraints,
         )
         await transaction.gather_requirements(requirements)
 
@@ -59,36 +62,31 @@ async def install(
                 f"See: {FAQ_URLS['cant_find_wheel']}\n"
             )
 
-        package_names = [pkg.name for pkg in transaction.pyodide_packages] + [
-            pkg.name for pkg in transaction.wheels
-        ]
+        packages_by_name: dict[str, PackageMetadata | WheelInfo] = {
+            **{pkg.name: pkg for pkg in transaction.pyodide_packages},
+            **{pkg.name: pkg for pkg in transaction.wheels},
+        }
+
+        try:
+            wheel_tasks = _build_wheel_tasks(
+                transaction.dependency_graph, packages_by_name, wheel_base
+            )
+        except CycleError as err:
+            raise ValueError(
+                f"Requested transaction contains at least one cycle: {err}"
+            ) from err
 
         logger.debug(
             "Installing packages %r and wheels %r ",
             transaction.pyodide_packages,
             [w.filename for w in transaction.wheels],
         )
-        if package_names:
-            logger.info("Installing collected packages: %s", ", ".join(package_names))
-
-        wheel_promises: list[Coroutine[Any, Any, None] | asyncio.Task[Any]] = []
-        # Install built-in packages
-        pyodide_packages = transaction.pyodide_packages
-        if len(pyodide_packages):
-            # Note: branch never happens in out-of-browser testing because in
-            # that case REPODATA_PACKAGES is empty.
-            wheel_promises.append(
-                asyncio.ensure_future(
-                    loadPackage(to_js([name for [name, _, _] in pyodide_packages]))
-                )
+        if packages_by_name:
+            logger.info(
+                "Installing collected packages: %s", ", ".join(packages_by_name)
             )
 
-        # Now install PyPI packages
-        # detect whether the wheel metadata is from PyPI or from custom location
-        # wheel metadata from PyPI has SHA256 checksum digest.
-        wheel_promises.extend(wheel.install(wheel_base) for wheel in transaction.wheels)
-
-        await asyncio.gather(*wheel_promises)
+        await asyncio.gather(*wheel_tasks.values())
 
         packages = [
             f"{pkg.name}-{pkg.version}" for pkg in transaction.pyodide_packages
@@ -98,3 +96,55 @@ async def install(
             logger.info("Successfully installed %s", ", ".join(packages))
 
         importlib.invalidate_caches()
+
+
+def _build_wheel_tasks(
+    graph: dict[str, list[str]],
+    packages_by_name: dict[str, PackageMetadata | WheelInfo],
+    wheel_base: Path,
+) -> dict[str, asyncio.Task[None]]:
+    """Build a verified task graph.
+
+    Raises a ``CycleError`` if any cycles are found.
+    """
+    sorted_graph = TopologicalSorter(graph)
+    sorted_graph.prepare()
+
+    tasks: dict[str, asyncio.Task[None]] = {}
+
+    for pkg_name in packages_by_name:
+        tasks[pkg_name] = _install_one(
+            pkg_name,
+            packages_by_name,
+            graph,
+            tasks,
+            wheel_base,
+        )
+
+    return tasks
+
+
+def _install_one(
+    pkg_name: str,
+    packages_by_name: dict[str, PackageMetadata | WheelInfo],
+    dependency_graph: dict[str, list[str]],
+    wheel_tasks: dict[str, asyncio.Task[None]],
+    wheel_base: Path,
+) -> asyncio.Task[None]:
+    """Build a task that installs a wheel after any dependencies are installeds."""
+
+    async def _install_one_inner():
+        wheel = packages_by_name.get(pkg_name)
+        dependencies = [
+            wheel_tasks[dependency]
+            for dependency in dependency_graph.get(pkg_name, [])
+            if dependency in wheel_tasks
+        ]
+        if dependencies:
+            await asyncio.gather(*dependencies)
+        if isinstance(wheel, WheelInfo):
+            await wheel.install(wheel_base)
+        elif isinstance(wheel, PackageMetadata):
+            await asyncio.ensure_future(loadPackage(to_js(wheel.name)))
+
+    return asyncio.Task(_install_one_inner(), name=pkg_name)