diff --git a/docs/changelog/3350.feature.rst b/docs/changelog/3350.feature.rst new file mode 100644 index 000000000..fea5c0907 --- /dev/null +++ b/docs/changelog/3350.feature.rst @@ -0,0 +1 @@ +Added ``constraints`` to allow specifying constraints files for all dependencies. diff --git a/docs/config.rst b/docs/config.rst index d9127b7ca..a741ed57d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -942,15 +942,15 @@ Python run :keys: deps :default: - Name of the Python dependencies. Installed into the environment prior to project after environment creation, but + Python dependencies. Installed into the environment prior to project after environment creation, but before package installation. All installer commands are executed using the :ref:`tox_root` as the current working directory. Each value must be one of: - a Python dependency as specified by :pep:`440`, - a `requirement file `_ when the value starts with - ``-r`` (followed by a file path), + ``-r`` (followed by a file path or URL), - a `constraint file `_ when the value starts with - ``-c`` (followed by a file path). + ``-c`` (followed by a file path or URL). If you are only defining :pep:`508` requirements (aka no pip requirement files), you should use :ref:`dependency_groups` instead. @@ -977,6 +977,21 @@ Python run -r requirements.txt -c constraints.txt + .. note:: + + :ref:`constraints` is the preferred way to specify constraints files since they will apply to package dependencies + also. + +.. conf:: + :keys: constraints + :default: + :version_added: 4.28.0 + + `Constraints files `_ to use during package and + dependency installation. Provided constraints files will be used when installing package dependencies and any + additional dependencies specified in :ref:`deps`, but will not be used when installing the package itself. + Each value must be a file path or URL. + .. conf:: :keys: use_develop, usedevelop :default: false @@ -1210,7 +1225,6 @@ Pip installer This command will be executed only if executing on Continuous Integrations is detected (for example set environment variable ``CI=1``) or if journal is active. - .. conf:: :keys: pip_pre :default: false @@ -1227,7 +1241,7 @@ Pip installer If ``constrain_package_deps`` is true, then tox will create and use ``{env_dir}{/}constraints.txt`` when installing package dependencies during ``install_package_deps`` stage. When this value is set to false, any conflicting package - dependencies will override explicit dependencies and constraints passed to ``deps``. + dependencies will override explicit dependencies and constraints passed to :ref:`deps`. .. conf:: :keys: use_frozen_constraints diff --git a/pyproject.toml b/pyproject.toml index be712b6df..7b33ca018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,8 +97,7 @@ test = [ "pytest-mock>=3.14", "pytest-xdist>=3.6.1", "re-assert>=1.1", - "setuptools>=75.3; python_version<='3.8'", - "setuptools>=75.8; python_version>'3.8'", + "setuptools>=75.8", "time-machine>=2.15; implementation_name!='pypy'", "wheel>=0.45.1", ] diff --git a/src/tox/tox.schema.json b/src/tox/tox.schema.json index 1a2c9d26f..27abee07f 100644 --- a/src/tox/tox.schema.json +++ b/src/tox/tox.schema.json @@ -289,7 +289,14 @@ }, "deps": { "type": "string", - "description": "Name of the python dependencies as specified by PEP-440" + "description": "python dependencies with optional version specifiers, as specified by PEP-440" + }, + "constraints": { + "type": "array", + "items": { + "type": "string" + }, + "description": "constraints to apply to installed python dependencies" }, "dependency_groups": { "type": "array", diff --git a/src/tox/tox_env/python/pip/pip_install.py b/src/tox/tox_env/python/pip/pip_install.py index 22e09f3a1..f36d0b9c9 100644 --- a/src/tox/tox_env/python/pip/pip_install.py +++ b/src/tox/tox_env/python/pip/pip_install.py @@ -4,8 +4,9 @@ import operator from abc import ABC, abstractmethod from collections import defaultdict +from functools import partial from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Sequence +from typing import TYPE_CHECKING, Any, Callable, Sequence, cast from packaging.requirements import Requirement @@ -15,7 +16,7 @@ from tox.tox_env.installer import Installer from tox.tox_env.python.api import Python from tox.tox_env.python.package import EditableLegacyPackage, EditablePackage, SdistPackage, WheelPackage -from tox.tox_env.python.pip.req_file import PythonDeps +from tox.tox_env.python.pip.req_file import PythonConstraints, PythonDeps if TYPE_CHECKING: from tox.config.main import Config @@ -52,6 +53,7 @@ class Pip(PythonInstallerListDependencies): def _register_config(self) -> None: super()._register_config() + root = self._env.core["toxinidir"] self._env.conf.add_config( keys=["pip_pre"], of_type=bool, @@ -65,6 +67,13 @@ def _register_config(self) -> None: post_process=self.post_process_install_command, desc="command used to install packages", ) + self._env.conf.add_config( + keys=["constraints"], + of_type=PythonConstraints, + factory=partial(PythonConstraints.factory, root), + default=PythonConstraints("", root), + desc="constraints to apply to installed python dependencies", + ) self._env.conf.add_config( keys=["constrain_package_deps"], of_type=bool, @@ -110,6 +119,10 @@ def install(self, arguments: Any, section: str, of_type: str) -> None: logging.warning("pip cannot install %r", arguments) raise SystemExit(1) + @property + def constraints(self) -> PythonConstraints: + return cast("PythonConstraints", self._env.conf["constraints"]) + def constraints_file(self) -> Path: return Path(self._env.env_dir) / "constraints.txt" @@ -121,16 +134,25 @@ def constrain_package_deps(self) -> bool: def use_frozen_constraints(self) -> bool: return bool(self._env.conf["use_frozen_constraints"]) - def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None: # noqa: C901 + def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None: + new_requirements: list[str] = [] + new_constraints: list[str] = [] + try: new_options, new_reqs = arguments.unroll() except ValueError as exception: msg = f"{exception} for tox env py within deps" raise Fail(msg) from exception - new_requirements: list[str] = [] - new_constraints: list[str] = [] for req in new_reqs: (new_constraints if req.startswith("-c ") else new_requirements).append(req) + + try: + _, new_reqs = self.constraints.unroll() + except ValueError as exception: + msg = f"{exception} for tox env py within constraints" + raise Fail(msg) from exception + new_constraints.extend(new_reqs) + constraint_options = { "constrain_package_deps": self.constrain_package_deps, "use_frozen_constraints": self.use_frozen_constraints, @@ -159,17 +181,10 @@ def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type raise Recreate(msg) args = arguments.as_root_args if args: # pragma: no branch + args.extend(self.constraints.as_root_args) self._execute_installer(args, of_type) if self.constrain_package_deps and not self.use_frozen_constraints: - # when we drop Python 3.8 we can use the builtin `.removeprefix` - def remove_prefix(text: str, prefix: str) -> str: - if text.startswith(prefix): - return text[len(prefix) :] - return text - - combined_constraints = new_requirements + [ - remove_prefix(text=c, prefix="-c ") for c in new_constraints - ] + combined_constraints = new_requirements + [c.removeprefix("-c ") for c in new_constraints] self.constraints_file().write_text("\n".join(combined_constraints)) @staticmethod @@ -215,13 +230,18 @@ def _install_list_of_deps( # noqa: C901 raise Recreate(msg) # pragma: no branch new_deps = sorted(set(groups["req"]) - set(old or [])) if new_deps: # pragma: no branch + new_deps.extend(self.constraints.as_root_args) self._execute_installer(new_deps, req_of_type) install_args = ["--force-reinstall", "--no-deps"] if groups["pkg"]: + # we intentionally ignore constraints when installing the package itself + # https://github.com/tox-dev/tox/issues/3550 self._execute_installer(install_args + groups["pkg"], of_type) if groups["dev_pkg"]: for entry in groups["dev_pkg"]: install_args.extend(("-e", str(entry))) + # we intentionally ignore constraints when installing the package itself + # https://github.com/tox-dev/tox/issues/3550 self._execute_installer(install_args, of_type) def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None: diff --git a/src/tox/tox_env/python/pip/req_file.py b/src/tox/tox_env/python/pip/req_file.py index d1fc4e810..fe6dbba1b 100644 --- a/src/tox/tox_env/python/pip/req_file.py +++ b/src/tox/tox_env/python/pip/req_file.py @@ -56,7 +56,7 @@ def _is_url_self(self, url: str) -> bool: def _pre_process(self, content: str) -> ReqFileLines: for at, line in super()._pre_process(content): if line.startswith("-r") or (line.startswith("-c") and line[2].isalpha()): - found_line = f"{line[0:2]} {line[2:]}" + found_line = f"{line[0:2]} {line[2:]}" # normalize else: found_line = line yield at, found_line @@ -64,18 +64,18 @@ def _pre_process(self, content: str) -> ReqFileLines: def lines(self) -> list[str]: return self._raw.splitlines() - @staticmethod - def _normalize_raw(raw: str) -> str: + @classmethod + def _normalize_raw(cls, raw: str) -> str: # a line ending in an unescaped \ is treated as a line continuation and the newline following it is effectively # ignored raw = "".join(raw.replace("\r", "").split("\\\n")) # for tox<4 supporting requirement/constraint files via -rreq.txt/-creq.txt - lines: list[str] = [PythonDeps._normalize_line(line) for line in raw.splitlines()] + lines: list[str] = [cls._normalize_line(line) for line in raw.splitlines()] adjusted = "\n".join(lines) return f"{adjusted}\n" if raw.endswith("\\\n") else adjusted # preserve trailing newline if input has it - @staticmethod - def _normalize_line(line: str) -> str: + @classmethod + def _normalize_line(cls, line: str) -> str: arg_match = next( ( arg @@ -138,6 +138,113 @@ def factory(cls, root: Path, raw: object) -> PythonDeps: return cls(raw, root) +class PythonConstraints(RequirementsFile): + def __init__(self, raw: str | list[str] | list[Requirement], root: Path) -> None: + super().__init__(root / "tox.ini", constraint=True) + got = raw if isinstance(raw, str) else "\n".join(str(i) for i in raw) + self._raw = self._normalize_raw(got) + self._unroll: tuple[list[str], list[str]] | None = None + self._req_parser_: RequirementsFile | None = None + + @property + def _req_parser(self) -> RequirementsFile: + if self._req_parser_ is None: + self._req_parser_ = RequirementsFile(path=self._path, constraint=True) + return self._req_parser_ + + def _get_file_content(self, url: str) -> str: + if self._is_url_self(url): + return self._raw + return super()._get_file_content(url) + + def _is_url_self(self, url: str) -> bool: + return url == str(self._path) + + def _pre_process(self, content: str) -> ReqFileLines: + for at, line in super()._pre_process(content): + if line.startswith("-r") or (line.startswith("-c") and line[2].isalpha()): + found_line = f"{line[0:2]} {line[2:]}" # normalize + else: + found_line = line + yield at, found_line + + def lines(self) -> list[str]: + return self._raw.splitlines() + + @classmethod + def _normalize_raw(cls, raw: str) -> str: + # a line ending in an unescaped \ is treated as a line continuation and the newline following it is effectively + # ignored + raw = "".join(raw.replace("\r", "").split("\\\n")) + # for tox<4 supporting requirement/constraint files via -rreq.txt/-creq.txt + lines: list[str] = [cls._normalize_line(line) for line in raw.splitlines()] + + if any(line.startswith("-") for line in lines): + msg = "only constraints files or URLs can be provided" + raise ValueError(msg) + + adjusted = "\n".join([f"-c {line}" for line in lines]) + return f"{adjusted}\n" if raw.endswith("\\\n") else adjusted # preserve trailing newline if input has it + + @classmethod + def _normalize_line(cls, line: str) -> str: + arg_match = next( + ( + arg + for arg in ONE_ARG + if line.startswith(arg) + and len(line) > len(arg) + and not (line[len(arg)].isspace() or line[len(arg)] == "=") + ), + None, + ) + if arg_match is not None: + values = line[len(arg_match) :] + line = f"{arg_match} {values}" + # escape spaces + escape_match = next((e for e in ONE_ARG_ESCAPE if line.startswith(e) and line[len(e)].isspace()), None) + if escape_match is not None: + # escape not already escaped spaces + escaped = re.sub(r"(? list[ParsedRequirement]: # noqa: FBT001 + # check for any invalid options in the deps list + # (requirements recursively included from other files are not checked) + requirements = super()._parse_requirements(opt, recurse) + for req in requirements: + if req.from_file != str(self.path): + continue + if req.options: + msg = f"Cannot provide options in constraints list, only paths or URL can be provided. ({req})" + raise ValueError(msg) + return requirements + + def unroll(self) -> tuple[list[str], list[str]]: + if self._unroll is None: + opts_dict = vars(self.options) + if not self.requirements and opts_dict: + msg = "no dependencies" + raise ValueError(msg) + result_opts: list[str] = [f"{key}={value}" for key, value in opts_dict.items()] + result_req = [str(req) for req in self.requirements] + self._unroll = result_opts, result_req + return self._unroll + + @classmethod + def factory(cls, root: Path, raw: object) -> PythonConstraints: + if not ( + isinstance(raw, str) + or ( + isinstance(raw, list) + and (all(isinstance(i, str) for i in raw) or all(isinstance(i, Requirement) for i in raw)) + ) + ): + raise TypeError(raw) + return cls(raw, root) + + ONE_ARG = { "-i", "--index-url", diff --git a/src/tox/tox_env/python/runner.py b/src/tox/tox_env/python/runner.py index 1a666a7ad..5c012857d 100644 --- a/src/tox/tox_env/python/runner.py +++ b/src/tox/tox_env/python/runner.py @@ -34,11 +34,11 @@ def register_config(self) -> None: super().register_config() root = self.core["toxinidir"] self.conf.add_config( - keys="deps", + keys=["deps"], of_type=PythonDeps, factory=partial(PythonDeps.factory, root), default=PythonDeps("", root), - desc="Name of the python dependencies as specified by PEP-440", + desc="python dependencies with optional version specifiers, as specified by PEP-440", ) self.conf.add_config( keys=["dependency_groups"], diff --git a/src/tox/tox_env/runner.py b/src/tox/tox_env/runner.py index 50adb6a2a..9d9b61bb3 100644 --- a/src/tox/tox_env/runner.py +++ b/src/tox/tox_env/runner.py @@ -89,28 +89,29 @@ def interrupt(self) -> None: self._call_pkg_envs("interrupt") def get_package_env_types(self) -> tuple[str, str] | None: - if self._register_package_conf(): - has_external_pkg = self.conf["package"] == "external" - self.core.add_config( - keys=["package_env", "isolated_build_env"], - of_type=str, - default=self._default_package_env, - desc="tox environment used to package", - ) - self.conf.add_config( - keys=["package_env"], - of_type=str, - default=f"{self.core['package_env']}{'_external' if has_external_pkg else ''}", - desc="tox environment used to package", - ) - is_external = self.conf["package"] == "external" - self.conf.add_constant( - keys=["package_tox_env_type"], - desc="tox package type used to generate the package", - value=self._external_pkg_tox_env_type if is_external else self._package_tox_env_type, - ) - return self.conf["package_env"], self.conf["package_tox_env_type"] - return None + if not self._register_package_conf(): + return None + + has_external_pkg = self.conf["package"] == "external" + self.core.add_config( + keys=["package_env", "isolated_build_env"], + of_type=str, + default=self._default_package_env, + desc="tox environment used to package", + ) + self.conf.add_config( + keys=["package_env"], + of_type=str, + default=f"{self.core['package_env']}{'_external' if has_external_pkg else ''}", + desc="tox environment used to package", + ) + is_external = self.conf["package"] == "external" + self.conf.add_constant( + keys=["package_tox_env_type"], + desc="tox package type used to generate the package", + value=self._external_pkg_tox_env_type if is_external else self._package_tox_env_type, + ) + return self.conf["package_env"], self.conf["package_tox_env_type"] def _call_pkg_envs(self, method_name: str, *args: Any) -> None: for package_env in self.package_envs: diff --git a/tests/tox_env/python/pip/test_pip_install.py b/tests/tox_env/python/pip/test_pip_install.py index ca5aa6ca5..bc4c9c949 100644 --- a/tests/tox_env/python/pip/test_pip_install.py +++ b/tests/tox_env/python/pip/test_pip_install.py @@ -56,6 +56,30 @@ def test_pip_install_flags_only_error(tox_project: ToxProjectCreator) -> None: assert "no dependencies for tox env py within deps" in result.out +@pytest.mark.parametrize( + ("content", "error"), + [ + pytest.param( + "-r requirements.txt", + "only constraints files or URLs can be provided", + id="requirements file are not constraints files", + ), + pytest.param( + "-c constraints.txt", + "only constraints files or URLs can be provided", + id="constraints flag is unnecessary", + ), + pytest.param("tox==4.23.0", "Could not open requirements file", id="constraints must be specified in a file"), + ], +) +def test_pip_install_invalid_constraints_error(tox_project: ToxProjectCreator, content: str, error: str) -> None: + proj = tox_project({"tox.ini": f"[testenv:py]\nconstraints={content}\nskip_install=true"}) + + result = proj.run("r") + result.assert_failed() + assert error in result.out + + def test_pip_install_new_flag_recreates(tox_project: ToxProjectCreator) -> None: proj = tox_project({"tox.ini": "[testenv:py]\ndeps=a\nskip_install=true"}) proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) @@ -231,8 +255,17 @@ def test_pip_install_requirements_file_deps(tox_project: ToxProjectCreator) -> N assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", "b", "-r", "r.txt"] -def test_pip_install_constraint_file_create_change(tox_project: ToxProjectCreator) -> None: - proj = tox_project({"tox.ini": "[testenv]\ndeps=-c c.txt\n a\nskip_install=true", "c.txt": "b"}) +@pytest.fixture(params=[True, False]) +def use_constraints_opt(request: SubRequest) -> bool: + return bool(request.param) + + +def test_pip_install_constraint_file_create_change(tox_project: ToxProjectCreator, use_constraints_opt: bool) -> None: + if use_constraints_opt: + toxini = "[testenv]\ndeps=a\nconstraints=c.txt\nskip_install=true" + else: + toxini = "[testenv]\ndeps=-c c.txt\n a\nskip_install=true" + proj = tox_project({"tox.ini": toxini, "c.txt": "b"}) execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) result = proj.run("r") result.assert_success() @@ -240,7 +273,11 @@ def test_pip_install_constraint_file_create_change(tox_project: ToxProjectCreato assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", "a", "-c", "c.txt"] # a new dependency triggers an install - (proj.path / "tox.ini").write_text("[testenv]\ndeps=-c c.txt\n a\n d\nskip_install=true") + if use_constraints_opt: + toxini = "[testenv]\ndeps=a\n d\nconstraints=c.txt\nskip_install=true" + else: + toxini = "[testenv]\ndeps=-c c.txt\n a\n d\nskip_install=true" + (proj.path / "tox.ini").write_text(toxini) execute_calls.reset_mock() result_second = proj.run("r") result_second.assert_success() @@ -257,7 +294,7 @@ def test_pip_install_constraint_file_create_change(tox_project: ToxProjectCreato assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", "a", "d", "-c", "c.txt"] -def test_pip_install_constraint_file_new(tox_project: ToxProjectCreator) -> None: +def test_pip_install_constraint_file_new(tox_project: ToxProjectCreator, use_constraints_opt: bool) -> None: proj = tox_project({"tox.ini": "[testenv]\ndeps=a\nskip_install=true"}) execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) result = proj.run("r") @@ -265,8 +302,12 @@ def test_pip_install_constraint_file_new(tox_project: ToxProjectCreator) -> None assert execute_calls.call_count == 1 assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", "a"] + if use_constraints_opt: + toxini = "[testenv]\ndeps=a\nconstraints=c.txt\nskip_install=true" + else: + toxini = "[testenv]\ndeps=a\n -c c.txt\nskip_install=true" (proj.path / "c.txt").write_text("a") - (proj.path / "tox.ini").write_text("[testenv]\ndeps=a\n -c c.txt\nskip_install=true") + (proj.path / "tox.ini").write_text(toxini) execute_calls.reset_mock() result_second = proj.run("r") result_second.assert_success()