diff --git a/changelog.d/2062.feature.md b/changelog.d/2062.feature.md new file mode 100644 index 000000000..10fd00bfb --- /dev/null +++ b/changelog.d/2062.feature.md @@ -0,0 +1,6 @@ +`pip-compile` now supports a `--group` flag for reading `[dependency-groups]` +data from `pyproject.toml` files. The option accepts data in the form +`path/to/pyproject.toml:groupname`, or bare `groupname` to read from +`./pyproject.toml`. + +-- by {user}`sirosen`. diff --git a/changelog.d/2175.feature.md b/changelog.d/2175.feature.md new file mode 120000 index 000000000..7e20d35da --- /dev/null +++ b/changelog.d/2175.feature.md @@ -0,0 +1 @@ +2062.feature.md \ No newline at end of file diff --git a/piptools/_internal/_cli/__init__.py b/piptools/_internal/_cli/__init__.py new file mode 100644 index 000000000..768d0d62c --- /dev/null +++ b/piptools/_internal/_cli/__init__.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from ._param_types import DependencyGroupParamType +from ._parsed_param_types import ParsedDependencyGroupParam + +__all__ = ( + "DependencyGroupParamType", + "ParsedDependencyGroupParam", +) diff --git a/piptools/_internal/_cli/_param_types.py b/piptools/_internal/_cli/_param_types.py new file mode 100644 index 000000000..4346e2a0c --- /dev/null +++ b/piptools/_internal/_cli/_param_types.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import functools +import typing as _t + +import click + +from . import _parsed_param_types + +F = _t.TypeVar("F", bound=_t.Callable[..., _t.Any]) + + +def _add_ctx_arg(f: F) -> F: + """ + Make ``ParamType`` methods compatible with various click versions, as documented + in the click docs here: + https://click.palletsprojects.com/en/stable/support-multiple-versions/ + """ + + @functools.wraps(f) + def wrapper(*args: _t.Any, **kwargs: _t.Any) -> _t.Any: + # NOTE: this check is skipped in coverage as it requires a lower `click` version + # NOTE: once we have a coverage plugin for library version pragmas, we can make + # NOTE: this do the appropriate dispatch + if "ctx" not in kwargs: # pragma: no cover + kwargs["ctx"] = click.get_current_context(silent=True) + + return f(*args, **kwargs) + + return wrapper # type: ignore[return-value] + + +class DependencyGroupParamType(click.ParamType): + @_add_ctx_arg + def get_metavar( # type: ignore[override] + self, param: click.Parameter, ctx: click.Context + ) -> str: + return "[pyproject-path:]groupname" + + def convert( + self, value: str, param: click.Parameter | None, ctx: click.Context | None + ) -> _parsed_param_types.ParsedDependencyGroupParam: + """Parse a ``[dependency-groups]`` group reference.""" + return _parsed_param_types.ParsedDependencyGroupParam(value) diff --git a/piptools/_internal/_cli/_parsed_param_types.py b/piptools/_internal/_cli/_parsed_param_types.py new file mode 100644 index 000000000..f8679b2b7 --- /dev/null +++ b/piptools/_internal/_cli/_parsed_param_types.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import pathlib + +import click + + +class ParsedDependencyGroupParam: + """ + Parse a dependency group input, but retain the input value. + + Splits on the rightmost ":", and validates that the path (if present) ends + in ``pyproject.toml``. Defaults the path to ``pyproject.toml`` when one is not given. + + ``:`` cannot appear in dependency group names, so this is a safe and simple parse. + + The following conversions are expected:: + + 'foo' -> ('pyproject.toml', 'foo') + 'foo/pyproject.toml:bar' -> ('foo/pyproject.toml', 'bar') + """ + + def __init__(self, value: str) -> None: + self.input_arg = value + + path, sep, groupname = value.rpartition(":") + if not sep: + path = "pyproject.toml" + else: + # check for 'pyproject.toml' filenames using pathlib + if pathlib.PurePath(path).name != "pyproject.toml": + msg = "group paths use 'pyproject.toml' filenames" + raise click.UsageError(msg) + + self.path = path + self.group = groupname + + def __str__(self) -> str: + return self.input_arg diff --git a/piptools/_internal/_dependency_groups.py b/piptools/_internal/_dependency_groups.py new file mode 100644 index 000000000..fe17b547f --- /dev/null +++ b/piptools/_internal/_dependency_groups.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import typing as _t +from collections.abc import Iterable + +import click +from dependency_groups import DependencyGroupResolver +from pip._internal.req import InstallRequirement +from pip._vendor.packaging.requirements import Requirement + +from .._compat import _tomllib_compat +from . import _cli + + +def parse_dependency_groups( + params: tuple[_cli.ParsedDependencyGroupParam, ...], +) -> list[InstallRequirement]: + resolvers = _build_resolvers(param.path for param in params) + reqs: list[InstallRequirement] = [] + for param in params: + resolver = resolvers[param.path] + try: + reqs.extend( + InstallRequirement( + Requirement(str(req)), + comes_from=f"--group '{param}'", + ) + for req in resolver.resolve(param.group) + ) + except (ValueError, TypeError, LookupError) as e: + raise click.UsageError( + f"[dependency-groups] resolution failed for '{param.group}' " + f"from '{param.path}': {e}" + ) from e + return reqs + + +def _build_resolvers(paths: Iterable[str]) -> dict[str, _t.Any]: + resolvers = {} + for path in paths: + if path in resolvers: + continue + + pyproject = _load_pyproject(path) + if "dependency-groups" not in pyproject: + raise click.UsageError( + f"[dependency-groups] table was missing from '{path}'. " + "Cannot resolve '--group' option." + ) + raw_dependency_groups = pyproject["dependency-groups"] + if not isinstance(raw_dependency_groups, dict): + raise click.UsageError( + f"[dependency-groups] table was malformed in {path}. " + "Cannot resolve '--group' option." + ) + + resolvers[path] = DependencyGroupResolver(raw_dependency_groups) + return resolvers + + +def _load_pyproject(path: str) -> dict[str, _t.Any]: + try: + with open(path, "rb") as fp: + return _tomllib_compat.load(fp) + except FileNotFoundError: + raise click.UsageError(f"{path} not found. Cannot resolve '--group' option.") + except _tomllib_compat.TOMLDecodeError as e: + raise click.UsageError(f"Error parsing {path}: {e}") from e + except OSError as e: + raise click.UsageError(f"Error reading {path}: {e}") from e diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 7fba15612..587da9ff3 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -14,7 +14,7 @@ from pip._internal.utils.misc import redact_auth_from_url from .._compat import canonicalize_name, parse_requirements, tempfile_compat -from .._internal import _pip_api +from .._internal import _cli, _dependency_groups, _pip_api from ..build import ProjectMetadata, build_project_metadata from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError @@ -127,6 +127,7 @@ def _determine_linesep( @options.reuse_hashes @options.max_rounds @options.src_files +@options.group @options.build_isolation @options.emit_find_links @options.cache_dir @@ -171,6 +172,7 @@ def cli( generate_hashes: bool, reuse_hashes: bool, src_files: tuple[str, ...], + groups: tuple[_cli.ParsedDependencyGroupParam, ...], max_rounds: int, build_isolation: bool, emit_find_links: bool, @@ -219,7 +221,7 @@ def cli( "--only-build-deps cannot be used with any of --extra, --all-extras" ) - if len(src_files) == 0: + if len(src_files) == 0 and len(groups) == 0: for file_path in DEFAULT_REQUIREMENTS_FILES: if os.path.exists(file_path): src_files = (file_path,) @@ -245,7 +247,7 @@ def cli( os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE ) # An output file must be provided if there are multiple source files - elif len(src_files) > 1: + elif len(src_files) + len(groups) > 1: raise click.BadParameter( "--output-file is required if two or more input files are given." ) @@ -418,6 +420,9 @@ def cli( ) ) + # Parse `--group` dependency-groups and add them to constraints + constraints.extend(_dependency_groups.parse_dependency_groups(groups)) + # Parse all constraints from `--constraint` files for filename in constraint: constraints.extend( diff --git a/piptools/scripts/options.py b/piptools/scripts/options.py index 3037e5dc9..5b516ecca 100644 --- a/piptools/scripts/options.py +++ b/piptools/scripts/options.py @@ -9,6 +9,8 @@ from piptools.locations import CACHE_DIR, DEFAULT_CONFIG_FILE_NAMES from piptools.utils import UNSAFE_PACKAGES, override_defaults_from_config_file +from .._internal import _cli + _FC = _t.TypeVar("_FC", bound="_t.Callable[..., _t.Any] | click.Command") BuildTargetT = _t.Literal["sdist", "wheel", "editable"] @@ -276,6 +278,18 @@ def _get_default_option(option_name: str) -> _t.Any: type=click.Path(exists=True, allow_dash=True), ) +group = click.option( + "--group", + "groups", + type=_cli.DependencyGroupParamType(), + multiple=True, + help=( + 'Specify a named dependency-group from a "pyproject.toml" file. ' + 'If a path is given, the name of the file must be "pyproject.toml". ' + 'Defaults to using "pyproject.toml" in the current directory.' + ), +) + build_isolation = click.option( "--build-isolation/--no-build-isolation", is_flag=True, diff --git a/piptools/utils.py b/piptools/utils.py index ecdc049e3..8135b9a83 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -27,7 +27,7 @@ from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from ._compat import _tomllib_compat, canonicalize_name -from ._internal import _subprocess +from ._internal import _cli, _subprocess _KT = _t.TypeVar("_KT") _VT = _t.TypeVar("_VT") @@ -375,6 +375,10 @@ def get_compile_command(click_ctx: click.Context) -> str: value = [value] for val in value: + # use the input value for --group params + if isinstance(val, _cli.ParsedDependencyGroupParam): + val = val.input_arg + # Flags don't have a value, thus add to args true or false option long name if option.is_flag: # If there are false-options, choose an option name depending on a value diff --git a/pyproject.toml b/pyproject.toml index e859deed6..795f3ed66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "build >= 1.0.0", "click >= 8", "pip >= 22.2", + "dependency-groups >= 1.3.0", "pyproject_hooks", "tomli; python_version < '3.11'", # indirect dependencies diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 9f7e1d1c3..ce13e0300 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1570,6 +1570,216 @@ def __exit__(self, *args, **kwargs): """) +@pytest.mark.parametrize("groupspec", ("mygroup", "pyproject.toml:mygroup")) +def test_dependency_group_resolution(pip_conf, runner, tmpdir_cwd, groupspec): + """ + Test compile requirements from pyproject.toml [dependency-groups]. + """ + pyproj = tmpdir_cwd / "pyproject.toml" + pyproj.write_text(dedent("""\ + [project] + name = "foo" + version = "1.0" + + [dependency-groups] + mygroup = ["small-fake-a==0.1"] + """)) + + out = runner.invoke( + cli, + [ + "--group", + groupspec, + "--output-file", + "-", + "--quiet", + "--no-emit-options", + "--no-header", + ], + ) + assert out.exit_code == 0, out.stderr + + assert out.stdout == dedent(f"""\ + small-fake-a==0.1 + # via --group '{groupspec}' + """) + + +@pytest.mark.parametrize( + ("pyproject_content", "group_arg", "expect_error"), + ( + pytest.param(None, "mygroup", "pyproject.toml not found", id="missing-file"), + pytest.param( + """\ + [project] + name = "foo" + version = "1.0" + """, + "mygroup", + "[dependency-groups] table was missing from 'pyproject.toml'", + id="missing-table", + ), + pytest.param( + # the malformed-toml test case needs to be in a non-cwd directory + # so that it properly tests malformed file parsing in this context + # if it's in the cwd, pip-compile will attempt to parse `./pyproject.toml` first + # (which will fail in a separate location) + """\ + [project + name = "foo" + version = "1.0" + """, + "./foo/pyproject.toml:mygroup", + "Error parsing ./foo/pyproject.toml", + id="malformed-toml", + ), + pytest.param( + """\ + [project] + name = "foo" + version = "1.0" + [dependency-groups] + mygroup = [{include-group = "mygroup"}] + """, + "mygroup", + "[dependency-groups] resolution failed for 'mygroup' from 'pyproject.toml'", + id="cyclic", + ), + pytest.param( + """\ + [project] + name = "foo" + version = "1.0" + [[dependency-groups]] + mygroup = "foo" + """, + "mygroup", + ( + "[dependency-groups] table was malformed in pyproject.toml. " + "Cannot resolve '--group' option." + ), + id="improper-table", + ), + ), +) +def test_dependency_group_resolution_fails_due_to_bad_data( + pip_conf, runner, tmpdir_cwd, pyproject_content, group_arg, expect_error +): + if ":" in group_arg: + path, _, _ = group_arg.partition(":") + (tmpdir_cwd / path).parent.mkdir(parents=True) + else: + path = "pyproject.toml" + pyproj = tmpdir_cwd / path + if pyproject_content is not None: + pyproj.write_text(dedent(pyproject_content)) + out = runner.invoke( + cli, + [ + "--group", + group_arg, + "--output-file", + "-", + "--quiet", + "--no-emit-options", + "--no-header", + ], + ) + assert out.exit_code == 2 + assert expect_error in out.stderr + + +def test_dependency_group_option_rejects_bad_filename(pip_conf, runner): + out = runner.invoke( + cli, + [ + "--group", + "my_file.toml:foo", + "--output-file", + "-", + "--quiet", + "--no-emit-options", + "--no-header", + ], + ) + assert out.exit_code == 2 + assert "group paths use 'pyproject.toml' filenames" in out.stderr + + +def test_dependency_group_option_exits_on_os_error(pip_conf, runner, tmpdir_cwd): + # IsADirectoryError is an OSError, so can be used to trip generic OSError handling + pyproject_dir = tmpdir_cwd / "pyproject.toml" + pyproject_dir.mkdir() + + out = runner.invoke( + cli, + [ + "--group", + "pyproject.toml:foo", + "--output-file", + "-", + "--quiet", + "--no-emit-options", + "--no-header", + ], + ) + assert out.exit_code == 2 + assert "Error reading pyproject.toml" in out.stderr + + +def test_dependency_group_resolution_with_multiple_files(pip_conf, runner, tmpdir_cwd): + """ + Compiling requirements from multiple pyproject.toml [dependency-groups] tables + can be done in a single resolution. + """ + for dirname in ("dir1", "dir2", "dir3"): + subdir = tmpdir_cwd / dirname + subdir.mkdir(parents=True) + pyproj = subdir / "pyproject.toml" + pyproj.write_text(dedent(f"""\ + [project] + name = "foo-{dirname}" + version = "1.0" + + [dependency-groups] + mygroup = ["small-fake-a==0.1"] + """)) + + group_args = [ + "--group", + "dir1/pyproject.toml:mygroup", + "--group", + "dir2/pyproject.toml:mygroup", + "--group", + "dir3/pyproject.toml:mygroup", + # repeats are also allowed + "--group", + "dir1/pyproject.toml:mygroup", + ] + + out = runner.invoke( + cli, + [ + *group_args, + "--output-file", + "-", + "--quiet", + "--no-emit-options", + "--no-header", + ], + ) + assert out.exit_code == 0, out.stderr + + # everything resolves to a single value, but the "via" sources are multiple + assert out.stdout == dedent("""\ + small-fake-a==0.1 + # via + # --group 'dir1/pyproject.toml:mygroup' + # --group 'dir2/pyproject.toml:mygroup' + # --group 'dir3/pyproject.toml:mygroup' + """) + + def test_multiple_input_files_without_output_file(runner): """ The --output-file option is required for multiple requirement input files. diff --git a/tests/test_writer.py b/tests/test_writer.py index dd4354485..495909de4 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import sys import pytest @@ -17,11 +18,11 @@ @pytest.fixture -def writer(tmpdir_cwd): +def writer_cli_args(tmpdir_cwd): with open("src_file", "w"), open("src_file2", "w"): pass - cli_args = [ + return [ "--dry-run", "--output-file", "requirements.txt", @@ -29,28 +30,41 @@ def writer(tmpdir_cwd): "src_file2", ] - with cli.make_context("pip-compile", cli_args) as ctx: - writer = OutputWriter( - dst_file=ctx.params["output_file"], - click_ctx=ctx, - dry_run=True, - emit_header=True, - emit_index_url=True, - emit_trusted_host=True, - annotate=True, - annotation_style="split", - generate_hashes=False, - default_index_url=None, - index_urls=[], - trusted_hosts=[], - format_control=FormatControl(set(), set()), - linesep="\n", - allow_unsafe=False, - find_links=[], - emit_find_links=True, - strip_extras=False, - emit_options=True, - ) + +@pytest.fixture +def writer_context(writer_cli_args): + @contextlib.contextmanager + def func(): + with cli.make_context("pip-compile", writer_cli_args) as ctx: + writer = OutputWriter( + dst_file=ctx.params["output_file"], + click_ctx=ctx, + dry_run=True, + emit_header=True, + emit_index_url=True, + emit_trusted_host=True, + annotate=True, + annotation_style="split", + generate_hashes=False, + default_index_url=None, + index_urls=[], + trusted_hosts=[], + format_control=FormatControl(set(), set()), + linesep="\n", + allow_unsafe=False, + find_links=[], + emit_find_links=True, + strip_extras=False, + emit_options=True, + ) + yield writer + + return func + + +@pytest.fixture +def writer(tmpdir_cwd, writer_context): + with writer_context() as writer: yield writer @@ -222,6 +236,28 @@ def test_write_header(writer): assert list(writer.write_header()) == list(expected) +@pytest.mark.parametrize("groupspec", ("mygroup", "pyproject.toml:mygroup")) +def test_write_header_group_opt(writer_cli_args, writer_context, groupspec): + writer_cli_args.clear() + writer_cli_args.extend(["--dry-run", "--group", groupspec]) + + expected = map( + comment, + [ + "#", + "# This file is autogenerated by pip-compile with Python " + f"{sys.version_info.major}.{sys.version_info.minor}", + "# by the following command:", + "#", + f"# pip-compile --group={groupspec}", + "#", + ], + ) + + with writer_context() as writer: + assert list(writer.write_header()) == list(expected) + + def test_write_header_custom_compile_command(writer, monkeypatch): monkeypatch.setenv("CUSTOM_COMPILE_COMMAND", "./pipcompilewrapper") expected = map(