True TOML config support

Signed-off-by: Bernát Gábor <[email protected]>

Author: gaborbernat

docs/changelog/999.feature.rst

@@ -0,0 +1 @@
+Native TOML configuration support - by :user:`gaborbernat`.

pyproject.toml

@@ -59,6 +59,7 @@ dependencies = [
   "pluggy>=1.5",
   "pyproject-api>=1.7.1",
   "tomli>=2.0.1; python_version<'3.11'",
+  "typing-extensions>=4.12.2; python_version<'3.11'",
   "virtualenv>=20.26.3",
 ]
 optional-dependencies.docs = [

src/tox/config/loader/toml.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Mapping,
+    Set,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from tox.config.loader.api import Loader, Override
+from tox.config.types import Command, EnvList
+from tox.report import HandledError
+
+if TYPE_CHECKING:
+    from tox.config.loader.section import Section
+    from tox.config.main import Config
+
+if sys.version_info >= (3, 11):  # pragma: no cover (py311+)
+    from typing import TypeGuard
+else:  # pragma: no cover (py311+)
+    from typing_extensions import TypeGuard
+if sys.version_info >= (3, 10):  # pragma: no cover (py310+)
+    from typing import TypeAlias
+else:  # pragma: no cover (py310+)
+    from typing_extensions import TypeAlias
+
+TomlTypes: TypeAlias = Union[Dict[str, "TomlTypes"], List["TomlTypes"], str, int, float, bool, None]
+
+
+class TomlLoader(Loader[TomlTypes]):
+    """Load configuration from a pyproject.toml file."""
+
+    def __init__(
+        self,
+        section: Section,
+        overrides: list[Override],
+        content: Mapping[str, TomlTypes],
+    ) -> None:
+        if not isinstance(content, Mapping):
+            msg = f"tox.{section.key} must be a mapping"
+            raise HandledError(msg)
+        self.content = content
+        super().__init__(section, overrides)
+
+    def load_raw(self, key: str, conf: Config | None, env_name: str | None) -> TomlTypes:  # noqa: ARG002
+        return self.content[key]
+
+    def found_keys(self) -> set[str]:
+        return set(self.content.keys())
+
+    @staticmethod
+    def to_str(value: TomlTypes) -> str:
+        return _ensure_type_correct(value, str)  # type: ignore[return-value] # no mypy support
+
+    @staticmethod
+    def to_bool(value: TomlTypes) -> bool:
+        return _ensure_type_correct(value, bool)
+
+    @staticmethod
+    def to_list(value: TomlTypes, of_type: type[Any]) -> Iterator[_T]:
+        of = List[of_type]  # type: ignore[valid-type] # no mypy support
+        return iter(_ensure_type_correct(value, of))  # type: ignore[call-overload,no-any-return]
+
+    @staticmethod
+    def to_set(value: TomlTypes, of_type: type[Any]) -> Iterator[_T]:
+        of = Set[of_type]  # type: ignore[valid-type] # no mypy support
+        return iter(_ensure_type_correct(value, of))  # type: ignore[call-overload,no-any-return]
+
+    @staticmethod
+    def to_dict(value: TomlTypes, of_type: tuple[type[Any], type[Any]]) -> Iterator[tuple[_T, _T]]:
+        of = Mapping[of_type[0], of_type[1]]  # type: ignore[valid-type] # no mypy support
+        return _ensure_type_correct(value, of).items()  # type: ignore[type-abstract,attr-defined,no-any-return]
+
+    @staticmethod
+    def to_path(value: TomlTypes) -> Path:
+        return Path(TomlLoader.to_str(value))
+
+    @staticmethod
+    def to_command(value: TomlTypes) -> Command:
+        return Command(args=cast(list[str], value))  # validated during load in _ensure_type_correct
+
+    @staticmethod
+    def to_env_list(value: TomlTypes) -> EnvList:
+        return EnvList(envs=list(TomlLoader.to_list(value, str)))
+
+
+_T = TypeVar("_T")
+
+
+def _ensure_type_correct(val: TomlTypes, of_type: type[_T]) -> TypeGuard[_T]:  # noqa: C901, PLR0912
+    casting_to = getattr(of_type, "__origin__", of_type.__class__)
+    msg = ""
+    if casting_to in {list, List}:
+        entry_type = of_type.__args__[0]  # type: ignore[attr-defined]
+        if not (isinstance(val, list) and all(_ensure_type_correct(v, entry_type) for v in val)):
+            msg = f"{val} is not list"
+    elif issubclass(of_type, Command):
+        # first we cast it to list then create commands, so for now just validate is a nested list
+        _ensure_type_correct(val, list[str])
+    elif casting_to in {set, Set}:
+        entry_type = of_type.__args__[0]  # type: ignore[attr-defined]
+        if not (isinstance(val, set) and all(_ensure_type_correct(v, entry_type) for v in val)):
+            msg = f"{val} is not set"
+    elif casting_to in {dict, Dict}:
+        key_type, value_type = of_type.__args__[0], of_type.__args__[1]  # type: ignore[attr-defined]
+        if not (
+            isinstance(val, dict)
+            and all(
+                _ensure_type_correct(dict_key, key_type) and _ensure_type_correct(dict_value, value_type)
+                for dict_key, dict_value in val.items()
+            )
+        ):
+            msg = f"{val} is not dictionary"
+    elif casting_to == Union:  # handle Optional values
+        args: list[type[Any]] = of_type.__args__  # type: ignore[attr-defined]
+        for arg in args:
+            try:
+                _ensure_type_correct(val, arg)
+                break
+            except TypeError:
+                pass
+        else:
+            msg = f"{val} is not union of {args}"
+    elif casting_to in {Literal, type(Literal)}:
+        choice = of_type.__args__  # type: ignore[attr-defined]
+        if val not in choice:
+            msg = f"{val} is not one of literal {choice}"
+    elif not isinstance(val, of_type):
+        msg = f"{val} is not one of {of_type}"
+    if msg:
+        raise TypeError(msg)
+    return cast(_T, val)  # type: ignore[return-value] # logic too complicated for mypy
+
+
+__all__ = [
+    "TomlLoader",
+]

src/tox/config/source/discover.py

@@ -9,12 +9,18 @@
 
 from .legacy_toml import LegacyToml
 from .setup_cfg import SetupCfg
+from .toml import Toml
 from .tox_ini import ToxIni
 
 if TYPE_CHECKING:
     from .api import Source
 
-SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, SetupCfg, LegacyToml)
+SOURCE_TYPES: tuple[type[Source], ...] = (
+    ToxIni,
+    SetupCfg,
+    LegacyToml,
+    Toml,
+)
 
 
 def discover_source(config_file: Path | None, root_dir: Path | None) -> Source:
@@ -79,7 +85,8 @@ def _create_default_source(root_dir: Path | None) -> Source:
                 break
     else:  # if not set use where we find pyproject.toml in the tree or cwd
         empty = root_dir
-    logging.warning("No %s found, assuming empty tox.ini at %s", " or ".join(i.FILENAME for i in SOURCE_TYPES), empty)
+    names = " or ".join({i.FILENAME: None for i in SOURCE_TYPES})
+    logging.warning("No %s found, assuming empty tox.ini at %s", names, empty)
     return ToxIni(empty / "tox.ini", content="")
 
 

src/tox/config/source/toml.py

@@ -0,0 +1,107 @@
+"""Load."""
+
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING, Any, Iterator, Mapping, cast
+
+from tox.config.loader.section import Section
+from tox.config.loader.toml import TomlLoader
+
+from .api import Source
+
+if sys.version_info >= (3, 11):  # pragma: no cover (py311+)
+    import tomllib
+else:  # pragma: no cover (py311+)
+    import tomli as tomllib
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from pathlib import Path
+
+    from tox.config.loader.api import Loader, OverrideMap
+    from tox.config.sets import CoreConfigSet
+
+TEST_ENV_PREFIX = "env"
+
+
+class TomlSection(Section):
+    SEP = "."
+
+    @classmethod
+    def test_env(cls, name: str) -> TomlSection:
+        return cls(f"tox{cls.SEP}{name}", name)
+
+    @property
+    def is_test_env(self) -> bool:
+        return self.prefix == TEST_ENV_PREFIX
+
+    @property
+    def keys(self) -> Iterable[str]:
+        return self.key.split(self.SEP)
+
+
+class Toml(Source):
+    """Configuration sourced from a pyproject.toml files."""
+
+    FILENAME = "pyproject.toml"
+
+    def __init__(self, path: Path) -> None:
+        if path.name != self.FILENAME or not path.exists():
+            raise ValueError
+        with path.open("rb") as file_handler:
+            toml_content = tomllib.load(file_handler)
+        try:
+            content: Mapping[str, Any] = toml_content["tool"]["tox"]
+            if "legacy_tox_ini" in content:
+                msg = "legacy_tox_ini"
+                raise KeyError(msg)  # noqa: TRY301
+            self._content = content
+        except KeyError as exc:
+            raise ValueError(path) from exc
+        super().__init__(path)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.path!r})"
+
+    def get_core_section(self) -> Section:  # noqa: PLR6301
+        return TomlSection(prefix=None, name="tox")
+
+    def transform_section(self, section: Section) -> Section:  # noqa: PLR6301
+        return TomlSection(section.prefix, section.name)
+
+    def get_loader(self, section: Section, override_map: OverrideMap) -> Loader[Any] | None:
+        current = self._content
+        for at, key in enumerate(cast(TomlSection, section).keys):
+            if at == 0:
+                if key != "tox":
+                    msg = "Internal error, first key is not tox"
+                    raise RuntimeError(msg)
+            elif key in current:
+                current = current[key]
+            else:
+                return None
+        return TomlLoader(
+            section=section,
+            overrides=override_map.get(section.key, []),
+            content=current,
+        )
+
+    def envs(self, core_conf: CoreConfigSet) -> Iterator[str]:
+        yield from core_conf["env_list"]
+        yield from [i.key for i in self.sections()]
+
+    def sections(self) -> Iterator[Section]:
+        for env_name in self._content.get("env", {}):
+            yield TomlSection.from_key(env_name)
+
+    def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Section]:  # noqa: PLR6301, ARG002
+        yield from [TomlSection.from_key(b) for b in base]
+
+    def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]:  # noqa: PLR6301
+        return TomlSection.test_env(item), ["tox.env_base"], ["tox.pkgenv"]
+
+
+__all__ = [
+    "Toml",
+]

tests/config/loader/toml/test_toml_loader.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from tox.pytest import ToxProjectCreator
+
+
+def test_conf_in_legacy_toml(tox_project: ToxProjectCreator) -> None:
+    project = tox_project({
+        "pyproject.toml": """
+    [tool.tox]
+    env_list = [ "A", "B"]
+
+    [tool.tox.env_base]
+    description = "Do magical things"
+    commands = [
+        ["python", "--version"],
+        ["python", "-c", "import sys; print(sys.executable)"]
+    ]
+
+    [tool.tox.env.C]
+    description = "Do magical things in C"
+    commands = [
+        ["python", "--version"]
+    ]
+    """
+    })
+
+    outcome = project.run("c", "--core", "-k", "commands")
+    outcome.assert_success()
+
+    outcome = project.run("c", "-e", "C,3.13")
+    outcome.assert_success()

tests/config/source/test_toml.py

@@ -34,7 +34,7 @@ commands =
       --cov-report html:{envtmpdir}{/}htmlcov \
       --cov-report xml:{toxworkdir}{/}coverage.{envname}.xml \
       -n={env:PYTEST_XDIST_AUTO_NUM_WORKERS:auto} \
-      tests --durations 5 --run-integration}
+      tests --durations 15 --run-integration}
     diff-cover --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}{/}coverage.{envname}.xml
 
 [testenv:fix]