Use PEP-621 to load project dependencies (#2499)

Author: gaborbernat

docs/changelog/2499.feature.rst

@@ -0,0 +1 @@
+Support PEP-621 static metadata for getting package dependencies - by :user:`gaborbernat`.

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
   "platformdirs>=2.5.2",
   "pluggy>=1",
   "pyproject-api>=0.1.1",
-  "tomli>=2.0.1",
+  'tomli>=2.0.1;python_version<"3.11"',
   "virtualenv>=20.16.5",
   'importlib-metadata>=4.12; python_version < "3.8"',
   'typing-extensions>=4.3; python_version < "3.8"',

src/tox/config/source/legacy_toml.py

@@ -1,8 +1,13 @@
 from __future__ import annotations
 
+import sys
 from pathlib import Path
 
-import tomli
+if sys.version_info >= (3, 11):  # pragma: no cover (py311+)
+    import tomllib
+else:  # pragma: no cover (py311+)
+    import tomli as tomllib
+
 
 from .ini import IniSource
 
@@ -14,7 +19,7 @@ def __init__(self, path: Path):
         if path.name != self.FILENAME or not path.exists():
             raise ValueError
         with path.open("rb") as file_handler:
-            toml_content = tomli.load(file_handler)
+            toml_content = tomllib.load(file_handler)
         try:
             content = toml_content["tool"]["tox"]["legacy_tox_ini"]
         except KeyError:

src/tox/execute/pep517_backend.py

@@ -59,7 +59,7 @@ def local_execute(self, options: ExecuteOptions) -> tuple[LocalSubProcessExecute
                     self.is_alive = True
                     break
                 if b"failed to start backend" in status.err:
-                    from tox.tox_env.python.virtual_env.package.pep517 import ToxBackendFailed
+                    from tox.tox_env.python.virtual_env.package.pyproject import ToxBackendFailed
 
                     failure = BackendFailed(
                         result={

src/tox/plugin/manager.py

@@ -13,7 +13,7 @@
 from tox.session.cmd.run import parallel, sequential
 from tox.tox_env import package as package_api
 from tox.tox_env.python.virtual_env import runner
-from tox.tox_env.python.virtual_env.package import cmd_builder, pep517
+from tox.tox_env.python.virtual_env.package import cmd_builder, pyproject
 from tox.tox_env.register import REGISTER, ToxEnvRegister
 
 from ..execute import Outcome
@@ -39,7 +39,7 @@ def _register_plugins(self, inline: ModuleType | None) -> None:
             loader_api,
             provision,
             runner,
-            pep517,
+            pyproject,
             cmd_builder,
             legacy,
             version_flag,

src/tox/provision.py

@@ -58,11 +58,12 @@ def tox_add_option(parser: ArgumentParser) -> None:
 
 
 def provision(state: State) -> int | bool:
+    # remove the dev and marker to allow local development of the package
     state.conf.core.add_config(
         keys=["min_version", "minversion"],
         of_type=Version,
         # do not include local version specifier (because it's not allowed in version spec per PEP-440)
-        default=Version(current_version.split("+")[0]),
+        default=Version(current_version),
         desc="Define the minimal tox version required to run",
     )
     state.conf.core.add_config(

src/tox/pytest.py

@@ -314,7 +314,7 @@ def enable_pep517_backend_coverage() -> Iterator[None]:  # noqa: PT004
         yield  # pragma: no cover
         return  # pragma: no cover
     # the COV_ env variables needs to be passed on for the PEP-517 backend
-    from tox.tox_env.python.virtual_env.package.pep517 import Pep517VirtualEnvPackager
+    from tox.tox_env.python.virtual_env.package.pyproject import Pep517VirtualEnvPackager
 
     def default_pass_env(self: Pep517VirtualEnvPackager) -> list[str]:
         result = previous(self)

src/tox/session/cmd/run/single.py

@@ -12,7 +12,7 @@
 from tox.execute.api import Outcome, StdinSource
 from tox.tox_env.api import ToxEnv
 from tox.tox_env.errors import Fail, Skip
-from tox.tox_env.python.virtual_env.package.pep517 import ToxBackendFailed
+from tox.tox_env.python.virtual_env.package.pyproject import ToxBackendFailed
 from tox.tox_env.runner import RunToxEnv
 
 LOGGER = logging.getLogger(__name__)

src/tox/tox_env/python/virtual_env/package/cmd_builder.py

@@ -27,7 +27,7 @@
 from tox.tox_env.register import ToxEnvRegister
 from tox.tox_env.runner import RunToxEnv
 
-from .pep517 import Pep517VirtualEnvPackager
+from .pyproject import Pep517VirtualEnvPackager
 from .util import dependencies_with_extras
 
 if sys.version_info >= (3, 8):  # pragma: no cover (py38+)

src/tox/tox_env/python/virtual_env/package/pep517.py renamed to src/tox/tox_env/python/virtual_env/package/pyproject.py

@@ -30,6 +30,12 @@
     from importlib.metadata import Distribution, PathDistribution
 else:  # pragma: no cover (<py38)
     from importlib_metadata import Distribution, PathDistribution
+
+if sys.version_info >= (3, 11):  # pragma: no cover (py311+)
+    import tomllib
+else:  # pragma: no cover (py311+)
+    import tomli as tomllib
+
 ConfigSettings = Optional[Dict[str, Any]]
 
 
@@ -143,23 +149,14 @@ def _teardown(self) -> None:
 
     def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]:
         """build the package to install"""
+        deps = self._load_deps(for_env)
         of_type: str = for_env["package"]
-
-        reqs: list[Requirement] | None = None
-        if of_type == "wheel":
-            w_env = self._wheel_build_envs.get(for_env["wheel_build_env"])
-            if w_env is not None and w_env is not self:
-                with w_env.display_context(self._has_display_suspended):
-                    reqs = w_env.get_package_dependencies() if isinstance(w_env, Pep517VirtualEnvPackager) else []
-        if reqs is None:
-            reqs = self.get_package_dependencies()
-
-        extras: set[str] = for_env["extras"]
-        deps = dependencies_with_extras(reqs, extras)
         if of_type == "dev-legacy":
+            self.setup()
             deps = [*self.requires(), *self._frontend.get_requires_for_build_sdist().requires] + deps
             package: Package = DevLegacyPackage(self.core["tox_root"], deps)  # the folder itself is the package
         elif of_type == "sdist":
+            self.setup()
             with self._pkg_lock:
                 package = SdistPackage(self._frontend.build_sdist(sdist_directory=self.pkg_dir).sdist, deps)
         elif of_type == "wheel":
@@ -168,6 +165,7 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]:
                 with w_env.display_context(self._has_display_suspended):
                     return w_env.perform_packaging(for_env)
             else:
+                self.setup()
                 with self._pkg_lock:
                     path = self._frontend.build_wheel(
                         wheel_directory=self.pkg_dir,
@@ -179,6 +177,49 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]:
             raise TypeError(f"cannot handle package type {of_type}")  # pragma: no cover
         return [package]
 
+    def _load_deps(self, for_env: EnvConfigSet) -> list[Requirement]:
+        # first check if this is statically available via PEP-621
+        deps = self._load_deps_from_static(for_env)
+        if deps is None:
+            deps = self._load_deps_from_built_metadata(for_env)
+        return deps
+
+    def _load_deps_from_static(self, for_env: EnvConfigSet) -> list[Requirement] | None:
+        pyproject_file = self.core["package_root"] / "pyproject.toml"
+        if not pyproject_file.exists():  # check if it's static PEP-621 metadata
+            return None
+        with pyproject_file.open("rb") as file_handler:
+            pyproject = tomllib.load(file_handler)
+        if "project" not in pyproject:
+            return None  # is not a PEP-621 pyproject
+        project = pyproject["project"]
+        extras: set[str] = for_env["extras"]
+        for dynamic in project.get("dynamic", []):
+            if dynamic == "dependencies" or (extras and dynamic == "optional-dependencies"):
+                return None  # if any dependencies are dynamic we can just calculate all dynamically
+
+        deps: list[Requirement] = [Requirement(i) for i in project.get("dependencies", [])]
+        optional_deps = project.get("optional-dependencies", {})
+        for extra in extras:
+            deps.extend(Requirement(i) for i in optional_deps.get(extra, []))
+        return deps
+
+    def _load_deps_from_built_metadata(self, for_env: EnvConfigSet) -> list[Requirement]:
+        # dependencies might depend on the python environment we're running in => if we build a wheel use that env
+        # to calculate the package metadata, otherwise ourselves
+        of_type: str = for_env["package"]
+        reqs: list[Requirement] | None = None
+        if of_type == "wheel":  # wheel packages
+            w_env = self._wheel_build_envs.get(for_env["wheel_build_env"])
+            if w_env is not None and w_env is not self:
+                with w_env.display_context(self._has_display_suspended):
+                    reqs = w_env.get_package_dependencies() if isinstance(w_env, Pep517VirtualEnvPackager) else []
+        if reqs is None:
+            reqs = self.get_package_dependencies()
+        extras: set[str] = for_env["extras"]
+        deps = dependencies_with_extras(reqs, extras)
+        return deps
+
     def get_package_dependencies(self) -> list[Requirement]:
         with self._pkg_lock:
             if self._package_dependencies is None:  # pragma: no branch