Implement a --group flag for dependency-group parsing for pip-compile

changelog.d/2062.feature.md

@@ -0,0 +1,6 @@
+`pip-compile` now supports a `--group` flag for reading `[dependency-groups]`
+data from `pyproject.toml` files. The option accepts data in the form
+`path/to/pyproject.toml:groupname`, or bare `groupname` to read from
+`./pyproject.toml`.
+
+-- by {user}`sirosen`.

changelog.d/2175.feature.md

@@ -0,0 +1 @@
+2062.feature.md

piptools/_internal/_cli/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from ._param_types import DependencyGroupParamType
+from ._parsed_param_types import ParsedDependencyGroupParam
+
+__all__ = (
+    "DependencyGroupParamType",
+    "ParsedDependencyGroupParam",
+)

piptools/_internal/_cli/_param_types.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import functools
+import typing as _t
+
+import click
+
+from . import _parsed_param_types
+
+F = _t.TypeVar("F", bound=_t.Callable[..., _t.Any])
+
+
+def _add_ctx_arg(f: F) -> F:
+    """
+    Make ``ParamType`` methods compatible with various click versions, as documented
+    in the click docs here:
+    https://click.palletsprojects.com/en/stable/support-multiple-versions/
+    """
+
+    @functools.wraps(f)
+    def wrapper(*args: _t.Any, **kwargs: _t.Any) -> _t.Any:
+        # NOTE: this check is skipped in coverage as it requires a lower `click` version
+        # NOTE: once we have a coverage plugin for library version pragmas, we can make
+        # NOTE: this do the appropriate dispatch
+        if "ctx" not in kwargs:  # pragma: no cover
+            kwargs["ctx"] = click.get_current_context(silent=True)
+
+        return f(*args, **kwargs)
+
+    return wrapper  # type: ignore[return-value]
+
+
+class DependencyGroupParamType(click.ParamType):
+    @_add_ctx_arg
+    def get_metavar(  # type: ignore[override]
+        self, param: click.Parameter, ctx: click.Context
+    ) -> str:
+        return "[pyproject-path:]groupname"
+
+    def convert(
+        self, value: str, param: click.Parameter | None, ctx: click.Context | None
+    ) -> _parsed_param_types.ParsedDependencyGroupParam:
+        """Parse a ``[dependency-groups]`` group reference."""
+        return _parsed_param_types.ParsedDependencyGroupParam(value)

piptools/_internal/_cli/_parsed_param_types.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import pathlib
+
+import click
+
+
+class ParsedDependencyGroupParam:
+    """
+    Parse a dependency group input, but retain the input value.
+
+    Splits on the rightmost ":", and validates that the path (if present) ends
+    in ``pyproject.toml``. Defaults the path to ``pyproject.toml`` when one is not given.
+
+    ``:`` cannot appear in dependency group names, so this is a safe and simple parse.
+
+    The following conversions are expected::
+
+        'foo' -> ('pyproject.toml', 'foo')
+        'foo/pyproject.toml:bar' -> ('foo/pyproject.toml', 'bar')
+    """
+
+    def __init__(self, value: str) -> None:
+        self.input_arg = value
+
+        path, sep, groupname = value.rpartition(":")
+        if not sep:
+            path = "pyproject.toml"
+        else:
+            # check for 'pyproject.toml' filenames using pathlib
+            if pathlib.PurePath(path).name != "pyproject.toml":
+                msg = "group paths use 'pyproject.toml' filenames"
+                raise click.UsageError(msg)
+
+        self.path = path
+        self.group = groupname
+
+    def __str__(self) -> str:
+        return self.input_arg

piptools/_internal/_dependency_groups.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import typing as _t
+from collections.abc import Iterable
+
+import click
+from dependency_groups import DependencyGroupResolver
+from pip._internal.req import InstallRequirement
+from pip._vendor.packaging.requirements import Requirement
+
+from .._compat import _tomllib_compat
+from . import _cli
+
+
+def parse_dependency_groups(
+    params: tuple[_cli.ParsedDependencyGroupParam, ...],
+) -> list[InstallRequirement]:
+    resolvers = _build_resolvers(param.path for param in params)
+    reqs: list[InstallRequirement] = []
+    for param in params:
+        resolver = resolvers[param.path]
+        try:
+            reqs.extend(
+                InstallRequirement(
+                    Requirement(str(req)),
+                    comes_from=f"--group '{param}'",
+                )
+                for req in resolver.resolve(param.group)
+            )
+        except (ValueError, TypeError, LookupError) as e:
+            raise click.UsageError(
+                f"[dependency-groups] resolution failed for '{param.group}' "
+                f"from '{param.path}': {e}"
+            ) from e
+    return reqs
+
+
+def _build_resolvers(paths: Iterable[str]) -> dict[str, _t.Any]:
+    resolvers = {}
+    for path in paths:
+        if path in resolvers:
+            continue
+
+        pyproject = _load_pyproject(path)
+        if "dependency-groups" not in pyproject:
+            raise click.UsageError(
+                f"[dependency-groups] table was missing from '{path}'. "
+                "Cannot resolve '--group' option."
+            )
+        raw_dependency_groups = pyproject["dependency-groups"]
+        if not isinstance(raw_dependency_groups, dict):
+            raise click.UsageError(
+                f"[dependency-groups] table was malformed in {path}. "
+                "Cannot resolve '--group' option."
+            )
+
+        resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
+    return resolvers
+
+
+def _load_pyproject(path: str) -> dict[str, _t.Any]:
+    try:
+        with open(path, "rb") as fp:
+            return _tomllib_compat.load(fp)
+    except FileNotFoundError:
+        raise click.UsageError(f"{path} not found. Cannot resolve '--group' option.")
+    except _tomllib_compat.TOMLDecodeError as e:
+        raise click.UsageError(f"Error parsing {path}: {e}") from e
+    except OSError as e:
+        raise click.UsageError(f"Error reading {path}: {e}") from e

piptools/scripts/compile.py

@@ -14,7 +14,7 @@
 from pip._internal.utils.misc import redact_auth_from_url
 
 from .._compat import canonicalize_name, parse_requirements, tempfile_compat
-from .._internal import _pip_api
+from .._internal import _cli, _dependency_groups, _pip_api
 from ..build import ProjectMetadata, build_project_metadata
 from ..cache import DependencyCache
 from ..exceptions import NoCandidateFound, PipToolsError
@@ -127,6 +127,7 @@ def _determine_linesep(
 @options.reuse_hashes
 @options.max_rounds
 @options.src_files
+@options.group
 @options.build_isolation
 @options.emit_find_links
 @options.cache_dir
@@ -171,6 +172,7 @@ def cli(
     generate_hashes: bool,
     reuse_hashes: bool,
     src_files: tuple[str, ...],
+    groups: tuple[_cli.ParsedDependencyGroupParam, ...],
     max_rounds: int,
     build_isolation: bool,
     emit_find_links: bool,
@@ -219,7 +221,7 @@ def cli(
             "--only-build-deps cannot be used with any of --extra, --all-extras"
         )
 
-    if len(src_files) == 0:
+    if len(src_files) == 0 and len(groups) == 0:
         for file_path in DEFAULT_REQUIREMENTS_FILES:
             if os.path.exists(file_path):
                 src_files = (file_path,)
@@ -245,7 +247,7 @@ def cli(
                 os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE
             )
         # An output file must be provided if there are multiple source files
-        elif len(src_files) > 1:
+        elif len(src_files) + len(groups) > 1:
             raise click.BadParameter(
                 "--output-file is required if two or more input files are given."
             )
@@ -418,6 +420,9 @@ def cli(
                 )
             )
 
+    # Parse `--group` dependency-groups and add them to constraints
+    constraints.extend(_dependency_groups.parse_dependency_groups(groups))
+
     # Parse all constraints from `--constraint` files
     for filename in constraint:
         constraints.extend(

piptools/scripts/options.py

@@ -9,6 +9,8 @@
 from piptools.locations import CACHE_DIR, DEFAULT_CONFIG_FILE_NAMES
 from piptools.utils import UNSAFE_PACKAGES, override_defaults_from_config_file
 
+from .._internal import _cli
+
 _FC = _t.TypeVar("_FC", bound="_t.Callable[..., _t.Any] | click.Command")
 
 BuildTargetT = _t.Literal["sdist", "wheel", "editable"]
@@ -276,6 +278,18 @@ def _get_default_option(option_name: str) -> _t.Any:
     type=click.Path(exists=True, allow_dash=True),
 )
 
+group = click.option(
+    "--group",
+    "groups",
+    type=_cli.DependencyGroupParamType(),
+    multiple=True,
+    help=(
+        'Specify a named dependency-group from a "pyproject.toml" file. '
+        'If a path is given, the name of the file must be "pyproject.toml". '
+        'Defaults to using "pyproject.toml" in the current directory.'
+    ),
+)
+
 build_isolation = click.option(
     "--build-isolation/--no-build-isolation",
     is_flag=True,

piptools/utils.py

@@ -27,7 +27,7 @@
 from piptools.locations import DEFAULT_CONFIG_FILE_NAMES
 
 from ._compat import _tomllib_compat, canonicalize_name
-from ._internal import _subprocess
+from ._internal import _cli, _subprocess
 
 _KT = _t.TypeVar("_KT")
 _VT = _t.TypeVar("_VT")
@@ -375,6 +375,10 @@ def get_compile_command(click_ctx: click.Context) -> str:
             value = [value]
 
         for val in value:
+            # use the input value for --group params
+            if isinstance(val, _cli.ParsedDependencyGroupParam):
+                val = val.input_arg
+
             # Flags don't have a value, thus add to args true or false option long name
             if option.is_flag:
                 # If there are false-options, choose an option name depending on a value

pyproject.toml

@@ -41,6 +41,7 @@ dependencies = [
   "build >= 1.0.0",
   "click >= 8",
   "pip >= 22.2",
+  "dependency-groups >= 1.3.0",
   "pyproject_hooks",
   "tomli; python_version < '3.11'",
   # indirect dependencies