Use tox.env.XXX testenvs in TOML, add native pyproject.toml

Author: ziima

src/tox/config/loader/section.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from abc import ABC
-from typing import TYPE_CHECKING
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Generic, TypeVar
 
 if TYPE_CHECKING:
     import sys
@@ -12,32 +12,30 @@
         from typing_extensions import Self
 
 
-class Section(ABC):  # noqa: PLW1641
-    """tox configuration section."""
+X = TypeVar("X")
+
+
+class BaseSection(ABC, Generic[X]):
+    """Base class for tox configuration section."""
 
     SEP = ":"  #: string used to separate the prefix and the section in the key
 
-    def __init__(self, prefix: str | None, name: str) -> None:
+    def __init__(self, prefix: X, name: str) -> None:
         self._prefix = prefix
         self._name = name
 
     @classmethod
+    @abstractmethod
     def from_key(cls: type[Self], key: str) -> Self:
         """
         Create a section from a section key.
 
         :param key: the section key
         :return: the constructed section
         """
-        sep_at = key.find(cls.SEP)
-        if sep_at == -1:
-            prefix, name = None, key
-        else:
-            prefix, name = key[:sep_at], key[sep_at + 1 :]
-        return cls(prefix, name)
 
     @property
-    def prefix(self) -> str | None:
+    def prefix(self) -> X:
         """:return: the prefix of the section"""
         return self._prefix
 
@@ -47,9 +45,9 @@ def name(self) -> str:
         return self._name
 
     @property
+    @abstractmethod
     def key(self) -> str:
         """:return: the section key"""
-        return self.SEP.join(i for i in (self._prefix, self._name) if i is not None)
 
     def __str__(self) -> str:
         return self.key
@@ -64,6 +62,31 @@ def __eq__(self, other: object) -> bool:
         )
 
 
+# TODO: Merge this with IniSection?
+class Section(BaseSection[str | None]):
+    """tox configuration section."""
+
+    @classmethod
+    def from_key(cls: type[Self], key: str) -> Self:
+        """
+        Create a section from a section key.
+
+        :param key: the section key
+        :return: the constructed section
+        """
+        sep_at = key.find(cls.SEP)
+        if sep_at == -1:
+            prefix, name = None, key
+        else:
+            prefix, name = key[:sep_at], key[sep_at + 1 :]
+        return cls(prefix, name)
+
+    @property
+    def key(self) -> str:
+        """:return: the section key"""
+        return self.SEP.join(i for i in (self._prefix, self._name) if i is not None)
+
+
 __all__ = [
     "Section",
 ]

src/tox/config/source/api.py

@@ -14,6 +14,8 @@
     from tox.config.sets import ConfigSet, CoreConfigSet
 
 
+# TODO: Generic in Section class?
+# TODO: Use BaseSection instead of Section?
 class Source(ABC):
     """Source is able to return a configuration value (for either the core or per environment source)."""
 

src/tox/config/source/discover.py

@@ -9,16 +9,13 @@
 
 from .legacy_toml import LegacyToml
 from .setup_cfg import SetupCfg
-from .toml import ToxToml
+from .toml import PyProjectToml, ToxToml
 from .tox_ini import ToxIni
 
-# from .toml import PyProjectToml
-
 if TYPE_CHECKING:
     from .api import Source
 
-# SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, ToxToml, PyProjectToml, SetupCfg, LegacyToml)
-SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, ToxToml, SetupCfg, LegacyToml)
+SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, ToxToml, PyProjectToml, SetupCfg, LegacyToml)
 
 
 def discover_source(config_file: Path | None, root_dir: Path | None) -> Source:

src/tox/config/source/toml.py

@@ -7,15 +7,15 @@
 
 from collections import defaultdict
 from itertools import chain
-from typing import TYPE_CHECKING, Iterable, Iterator
+from typing import TYPE_CHECKING, Any, Iterable, Iterator
 
 import tomllib
 
 from tox.config.loader.ini.factor import find_envs
 from tox.config.loader.memory import MemoryLoader
 
 from .api import Source
-from .toml_section import CORE, PKG_ENV_PREFIX, TEST_ENV_PREFIX, TomlSection
+from .toml_section import BASE_TEST_ENV, CORE, PKG_ENV_PREFIX, TEST_ENV_PREFIX, TEST_ENV_ROOT, TomlSection
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -25,21 +25,40 @@
     from tox.config.sets import ConfigSet
 
 
+def _extract_section(raw: dict[str, Any], section: TomlSection) -> Any:
+    """Extract section from TOML decoded data."""
+    result = raw
+    for key in chain(section.prefix, (section.name,)):
+        if key in result:
+            result = result[key]
+        else:
+            return None
+    return result
+
+
 class TomlSource(Source):
     """Configuration sourced from a toml file (such as tox.toml).
 
     This is experimental API! Expect things to be broken.
     """
 
     CORE_SECTION = CORE
+    ROOT_KEY: str | None = None
 
     def __init__(self, path: Path, content: str | None = None) -> None:
         super().__init__(path)
         if content is None:
             if not path.exists():
-                raise ValueError
+                msg = f"Path {path} does not exist."
+                raise ValueError(msg)
             content = path.read_text()
-        self._raw = tomllib.loads(content)
+        data = tomllib.loads(content)
+        if self.ROOT_KEY:
+            if self.ROOT_KEY not in data:
+                msg = f"Section {self.ROOT_KEY} not found in {path}."
+                raise ValueError(msg)
+            data = data[self.ROOT_KEY]
+        self._raw = data
         self._section_mapping: defaultdict[str, list[str]] = defaultdict(list)
 
     def __repr__(self) -> str:
@@ -48,32 +67,30 @@ def __repr__(self) -> str:
     def transform_section(self, section: Section) -> Section:
         return TomlSection(section.prefix, section.name)
 
-    def get_loader(self, section: Section, override_map: OverrideMap) -> MemoryLoader | None:
-        # look up requested section name in the generative testenv mapping to find the real config source
-        for key in self._section_mapping.get(section.name) or []:
-            if section.prefix is None or TomlSection.from_key(key).prefix == section.prefix:
-                break
-        else:
-            # if no matching section/prefix is found, use the requested section key as-is (for custom prefixes)
-            key = section.key
-        if key in self._raw:
-            return MemoryLoader(
-                self._raw[key],
-                section=section,
-                overrides=override_map.get(section.key, []),
-            )
-        return None
+    def get_loader(self, section: TomlSection, override_map: OverrideMap) -> MemoryLoader | None:
+        result = _extract_section(self._raw, section)
+        if result is None:
+            return None
+
+        return MemoryLoader(
+            result,
+            section=section,
+            overrides=override_map.get(section.key, []),
+        )
 
     def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Section]:  # noqa: PLR6301
         for a_base in base:
-            section = TomlSection.from_key(a_base)
-            yield section  # the base specifier is explicit
-            if in_section.prefix is not None:  # no prefix specified, so this could imply our own prefix
-                yield TomlSection(in_section.prefix, a_base)
+            yield TomlSection(in_section.prefix, a_base)
 
-    def sections(self) -> Iterator[Section]:
+    def sections(self) -> Iterator[TomlSection]:
+        # TODO: just return core section and any `tox.env.XXX` sections which exist directly.
         for key in self._raw:
-            yield TomlSection.from_key(key)
+            section = TomlSection.from_key(key)
+            yield section
+            if section == self.CORE_SECTION:
+                test_env_data = _extract_section(self._raw, TEST_ENV_ROOT)
+                for env_name in test_env_data or {}:
+                    yield TomlSection(TEST_ENV_PREFIX, env_name)
 
     def envs(self, core_config: ConfigSet) -> Iterator[str]:
         seen = set()
@@ -102,8 +119,9 @@ def register_factors(envs: Iterable[str]) -> None:
         for section in self.sections():
             yield from self._discover_from_section(section, known_factors)
 
-    def _discover_from_section(self, section: Section, known_factors: set[str]) -> Iterator[str]:
-        for value in self._raw[section.key].values():
+    def _discover_from_section(self, section: TomlSection, known_factors: set[str]) -> Iterator[str]:
+        section_data = _extract_section(self._raw, section)
+        for value in (section_data or {}).values():
             if isinstance(value, bool):
                 # It's not a value with env definition.
                 continue
@@ -113,8 +131,8 @@ def _discover_from_section(self, section: Section, known_factors: set[str]) -> I
                 if set(env.split("-")) - known_factors:
                     yield env
 
-    def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]:  # noqa: PLR6301
-        return TomlSection.test_env(item), [TEST_ENV_PREFIX], [PKG_ENV_PREFIX]
+    def get_tox_env_section(self, item: str) -> tuple[TomlSection, list[str], list[str]]:  # noqa: PLR6301
+        return TomlSection.test_env(item), [BASE_TEST_ENV], [PKG_ENV_PREFIX]
 
     def get_core_section(self) -> TomlSection:
         return self.CORE_SECTION
@@ -129,12 +147,18 @@ class ToxToml(TomlSource):
     FILENAME = "tox.toml"
 
 
-# TODO: Section model is way too configparser precific for this to work easily.
-# class PyProjectToml(TomlSource):
-#     """Configuration sourced from a pyproject.toml file.
+class PyProjectToml(TomlSource):
+    """Configuration sourced from a pyproject.toml file.
 
-#     This is experimental API! Expect things to be broken.
-#     """
+    This is experimental API! Expect things to be broken.
+    """
 
-#     FILENAME = "pyproject.toml"
-#     CORE_SECTION = IniSection("tool", "tox")
+    FILENAME = "pyproject.toml"
+    ROOT_KEY = "tool"
+
+    def __init__(self, path: Path, content: str | None = None) -> None:
+        super().__init__(path, content)
+        core_data = _extract_section(self._raw, self.CORE_SECTION)
+        if core_data is not None and tuple(core_data.keys()) == ("legacy_tox_ini",):
+            msg = "pyproject.toml is in the legacy mode."
+            raise ValueError(msg)

src/tox/config/source/toml_section.py

@@ -1,26 +1,51 @@
 from __future__ import annotations
 
+from itertools import chain
+from typing import Sequence
+
 from tox.config.loader.ini.factor import extend_factors
-from tox.config.loader.section import Section
+from tox.config.loader.section import BaseSection
 
-# TODO: Rename to 'env'
-TEST_ENV_PREFIX = "testenv"
+# TODO: Shouldn't this be loaded from config?
+BASE_TEST_ENV = "testenv"
+TEST_ENV_PREFIX = ("tox", "env")
+# TODO: PKG_ENV_PREFIX?
 PKG_ENV_PREFIX = "pkgenv"
 
 
-# TODO: Duplicates IniSection
-class TomlSection(Section):
+# TODO: Duplicates IniSection API
+class TomlSection(BaseSection[Sequence[str]]):
+    def __init__(self, prefix: Sequence[str] | None, name: str) -> None:
+        super().__init__(tuple(prefix or ()), name)
+
+    @classmethod
+    def from_key(cls: type[TomlSection], key: str) -> TomlSection:
+        """
+        Create a section from a section key.
+
+        :param key: the section key
+        :return: the constructed section
+        """
+        chunks = key.split(cls.SEP)
+        return cls(chunks[:-1], chunks[-1])
+
+    @property
+    def key(self) -> str:
+        """:return: the section key"""
+        return self.SEP.join(chain(self._prefix, (self._name,)))
+
     @classmethod
     def test_env(cls, name: str) -> TomlSection:
         return cls(TEST_ENV_PREFIX, name)
 
     @property
     def is_test_env(self) -> bool:
-        return self.prefix == TEST_ENV_PREFIX
+        return self.prefix == TEST_ENV_PREFIX and self.name != BASE_TEST_ENV
 
     @property
     def names(self) -> list[str]:
         return list(extend_factors(self.name))
 
 
 CORE = TomlSection(None, "tox")
+TEST_ENV_ROOT = TomlSection(TEST_ENV_PREFIX[:-1], TEST_ENV_PREFIX[-1])

tests/config/cli/test_cli_ini.py

@@ -149,12 +149,13 @@ def test_cli_ini_with_interpolated(tmp_path: Path, monkeypatch: MonkeyPatch) ->
         pytest.param("tox.toml", "tox.toml", "[tox]", id="toml"),
         pytest.param("", "setup.cfg", "[tox:tox]", id="cfg-dir"),
         pytest.param("setup.cfg", "setup.cfg", "[tox:tox]", id="cfg"),
-        # pytest.param("", "pyproject.toml", '[tool.tox]', id="toml-dir"),
-        # pytest.param("pyproject.toml", "pyproject.toml", '[tool.tox]', id="toml"),
-        pytest.param("", "pyproject.toml", '[tool.tox]\nlegacy_tox_ini = """\n[tox]\n"""\n', id="toml-legacy-dir"),
-        pytest.param(
-            "pyproject.toml", "pyproject.toml", '[tool.tox]\nlegacy_tox_ini = """\n[tox]\n"""\n', id="toml-legacy"
-        ),
+        pytest.param("", "pyproject.toml", "[tool.tox]", id="toml-dir"),
+        pytest.param("pyproject.toml", "pyproject.toml", "[tool.tox]", id="toml"),
+        # TODO: Fix this
+        # pytest.param("", "pyproject.toml", '[tool.tox]\nlegacy_tox_ini = """\n[tox]\n"""\n', id="toml-legacy-dir"),
+        # pytest.param(
+        #     "pyproject.toml", "pyproject.toml", '[tool.tox]\nlegacy_tox_ini = """\n[tox]\n"""\n', id="toml-legacy"
+        # ),
     ],
 )
 def test_conf_arg(tmp_path: Path, conf_arg: str, filename: str, content: str) -> None:

tests/config/source/test_discover.py

@@ -9,8 +9,9 @@
 
 
 def out_no_src(path: Path) -> str:
+    # TODO: Deduplicate file names.
     return (
-        f"ROOT: No tox.ini or tox.toml or setup.cfg or pyproject.toml found, assuming empty tox.ini at {path}\n"
+        f"ROOT: No tox.ini or tox.toml or pyproject.toml or setup.cfg or pyproject.toml found, assuming empty tox.ini at {path}\n"
         f"default environments:\npy -> [no description]\n"
     )