Packaging inherits from pkgenv, deps and document tox 4 packaging changes (#2813)

Resolves #2543

Author: gaborbernat

docs/changelog/2543.doc.rst

@@ -0,0 +1 @@
+Document breaking changes with tox 4 and packaging environments - by :user:`gaborbernat`.

docs/changelog/2543.feature.rst

@@ -0,0 +1,3 @@
+Packaging environments now inherit from the ``pkgenv`` section, allowing to set all your packaging options in one place,
+and support the ``deps`` key to set additional dependencies that will be installed after ``pyprojec.toml`` static
+``requires`` but before backends dynamic requires - by :user:`gaborbernat`.

docs/upgrading.rst

@@ -222,3 +222,96 @@ This is best avoided by updating to non-legacy usage:
 
     # or, equivalently...
     $ tox r -e list
+
+Packaging environments
+----------------------
+
+Isolated environment on by default
+++++++++++++++++++++++++++++++++++
+``tox`` now always uses an isolated build environment when building your projects package. The previous flag to enable
+this called ``isolated_build`` has been removed.
+
+Packaging configuration and inheritance
++++++++++++++++++++++++++++++++++++++++
+Isolated build environments are tox environments themselves and may be configured on their own. Their name is defined
+as follows:
+
+- For source distributions this environment will match a virtual environment with the same python interpreter as tox is
+  using. The name of this environment will by default ``.pkg`` (can be changed via :ref:`package_env` config on a per
+  test environment basis).
+- For wheels (including editable wheels as defined by :pep:`660`) their name will be ``.pkg-<impl><python_version>``, so
+  for example if you're building a wheel for a Python 3.10 environment the packaging environment will be
+  ``.pkg-cpython311``  (can be changed via :ref:`wheel_build_env` config on a per test environment basis).
+
+To change a packaging environments settings you can use:
+
+.. code-block:: ini
+
+    [testenv:.pkg]
+    pass_env =
+        PKG_CONFIG
+        PKG_CONFIG_PATH
+        PKG_CONFIG_SYSROOT_DIR
+
+    [testenv:.pkg-cpython311]
+    pass_env =
+        PKG_CONFIG
+        PKG_CONFIG_PATH
+        PKG_CONFIG_SYSROOT_DIR
+
+Packaging environments no longer inherit their settings from the ``testenv`` section, as this caused issues when
+some test environment settings conflicted with packaging setting. However starting with ``tox>=4.2`` all packaging
+environments inherit from the ``pkgenv`` section, allowing you to define packaging common packaging settings in one
+central place, while still allowing you to override it when needed on a per package environment basis:
+
+.. code-block:: ini
+
+    [pkgenv]
+    pass_env =
+        PKG_CONFIG
+        PKG_CONFIG_PATH
+        PKG_CONFIG_SYSROOT_DIR
+
+    [testenv:.pkg-cpython311]
+    pass_env =
+        {[pkgenv]pass_env}
+        IS_311 = yes
+
+    [testenv:magic]
+    package = sdist
+    pass_env = {[pkgenv]pass_env}  # sdist install builds wheel -> need packaging settings
+
+Note that specific packaging environments are defined under ``testenv:.pkg`` and **not** ``pkgenv:.pkg``, this is due
+backwards compatibility.
+
+Universal wheels
+++++++++++++++++
+If your project builds universal wheels you can avoid using multiple build environments for each targeted python by
+setting :ref:`wheel_build_env` to the same packaging environment via:
+
+.. code-block:: ini
+
+    [testenv]
+    package = wheel
+    wheel_build_env = .pkg
+
+Editable mode
++++++++++++++
+``tox`` now defaults to using editable wheels when develop mode is enabled and the build backend supports it,
+as defined by :pep:`660` by setting :ref:`package` to ``editable``. In case the backend does not support it, will
+fallback to :ref:`package` to ``editable-legacy``, and invoke pip with ``-e``. In the later case will also print a
+message to make this setting explicit in your configuration (explicit better than implicit):
+
+.. code-block:: ini
+
+    [testenv:dev]
+    package = editable-legacy
+
+If you want to use the new standardized method to achieve the editable install effect you should ensure your backend
+version is above the version this feature was added to it, for example for setuptools:
+
+.. code-block:: ini
+
+    [testenv:dev]
+    deps = setuptools>=64
+    package = editable

src/tox/config/main.py

@@ -152,18 +152,18 @@ def get_env(
         :param loaders: loaders to use for this configuration (only used for creation)
         :return: the tox environments config
         """
-        section, base = self._src.get_tox_env_section(item)
+        section, base_test, base_pkg = self._src.get_tox_env_section(item)
         conf_set = self.get_section_config(
             section,
-            base=None if package else base,
+            base=base_pkg if package else base_test,
             of_type=EnvConfigSet,
             for_env=item,
             loaders=loaders,
         )
         return conf_set
 
     def clear_env(self, name: str) -> None:
-        section, _ = self._src.get_tox_env_section(name)
+        section, _, __ = self._src.get_tox_env_section(name)
         del self._key_to_conf_set[(section.key, name)]
 
 

src/tox/config/source/api.py

@@ -98,7 +98,7 @@ def envs(self, core_conf: CoreConfigSet) -> Iterator[str]:
         raise NotImplementedError
 
     @abstractmethod
-    def get_tox_env_section(self, item: str) -> tuple[Section, list[str]]:
+    def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]:
         """:returns: the section for a tox environment"""
         raise NotImplementedError
 

src/tox/config/source/ini.py

@@ -14,7 +14,7 @@
 from ..loader.section import Section
 from ..sets import ConfigSet
 from .api import Source
-from .ini_section import CORE, TEST_ENV_PREFIX, IniSection
+from .ini_section import CORE, PKG_ENV_PREFIX, TEST_ENV_PREFIX, IniSection
 
 
 class IniSource(Source):
@@ -62,8 +62,8 @@ def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Se
             if in_section.prefix is not None:  # no prefix specified, so this could imply our own prefix
                 yield IniSection(in_section.prefix, a_base)
 
-    def get_tox_env_section(self, item: str) -> tuple[Section, list[str]]:
-        return IniSection.test_env(item), [TEST_ENV_PREFIX]
+    def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]:
+        return IniSection.test_env(item), [TEST_ENV_PREFIX], [PKG_ENV_PREFIX]
 
     def envs(self, core_config: ConfigSet) -> Iterator[str]:
         seen = set()

src/tox/config/source/ini_section.py

@@ -20,10 +20,12 @@ def names(self) -> list[str]:
 
 
 TEST_ENV_PREFIX = "testenv"
+PKG_ENV_PREFIX = "pkgenv"
 CORE = IniSection(None, "tox")
 
 __all__ = [
     "IniSection",
     "CORE",
     "TEST_ENV_PREFIX",
+    "PKG_ENV_PREFIX",
 ]

src/tox/tox_env/python/package.py

@@ -5,7 +5,7 @@
 
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generator, Iterator, Sequence, cast
+from typing import TYPE_CHECKING, Any, Generator, Iterator, List, Sequence, cast
 
 from packaging.requirements import Requirement
 
@@ -56,13 +56,22 @@ def _setup_env(self) -> None:
         """setup the tox environment"""
         super()._setup_env()
         self._install(self.requires(), PythonPackageToxEnv.__name__, "requires")
+        self._install(self.conf["deps"], PythonPackageToxEnv.__name__, "deps")
 
     @abstractmethod
     def requires(self) -> tuple[Requirement, ...] | PythonDeps:
         raise NotImplementedError
 
     def register_run_env(self, run_env: RunToxEnv) -> Generator[tuple[str, str], PackageToxEnv, None]:
         yield from super().register_run_env(run_env)
+        if run_env.conf["package"] != "skip" and "deps" not in self.conf:
+            self.conf.add_config(
+                keys="deps",
+                of_type=List[Requirement],
+                default=[],
+                desc="Name of the python dependencies as specified by PEP-440",
+            )
+
         if (
             not isinstance(run_env, Python)
             or run_env.conf["package"] not in {"wheel", "editable"}

src/tox/util/ci.py

@@ -21,7 +21,12 @@
 
 def is_ci() -> bool:
     """:return: a flag indicating if running inside a CI env or not"""
-    return any(e in os.environ if v is None else os.environ.get(e) == v for e, v in _ENV_VARS.items())
+    for env_key, value in _ENV_VARS.items():
+        if env_key in os.environ if value is None else os.environ.get(env_key) == value:
+            if env_key == "TEAMCITY_VERSION" and os.environ.get(env_key) == "LOCAL":
+                continue
+            return True
+    return False
 
 
 __all__ = [

tests/session/cmd/test_show_config.py

@@ -254,3 +254,20 @@ def test_show_config_matching_env_section(tox_project: ToxProjectCreator) -> Non
     outcome = project.run("c", "-e", "a,b", "-k", "deps")
     outcome.assert_success()
     assert outcome.out.count("c>=1") == 2, outcome.out
+
+
+def test_package_env_inherits_from_pkgenv(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None:
+    project = tox_project({"tox.ini": "[pkgenv]\npass_env = A, B\ndeps=C\n D"})
+    outcome = project.run("c", "--root", str(demo_pkg_inline), "-k", "deps", "pass_env", "-e", "py,.pkg")
+    outcome.assert_success()
+    exp = """
+    [testenv:.pkg]
+    deps =
+      C
+      D
+    pass_env =
+      A
+      B
+    """
+    exp = dedent(exp)
+    assert exp in outcome.out