Update --group option to take [path:]name

In discussions about the correct interface for `pip` to use
[dependency-groups], no strong consensus arose. However, the option
with the most support appears to be to make it possible to pass a file
path plus a group name.

This change converts the `--group` option to take colon-separated
path:groupname pairs, with the path part optional. The CLI parsing
code is responsible for handling the syntax and for filling in a
default path of `"pyproject.toml"`.

If a path is provided, it must have a basename of `pyproject.toml`.
Failing to meet this constraint is an error at arg parsing time.

The `dependency_groups` usage is updated to create a
DependencyGroupResolver per `pyproject.toml` file provided. This
ensures that we only parse each file once, and we keep the results of
previous resolutions when resolving multiple dependency groups from
the same file. (Technically, the implementation is a resolver per
path, which is subtly different from per-file, in that it doesn't
account for symlinks, hardlinks, etc.)

Author: sirosen

src/pip/_internal/cli/cmdoptions.py

@@ -13,6 +13,7 @@
 import importlib.util
 import logging
 import os
+import pathlib
 import textwrap
 from functools import partial
 from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values
@@ -733,15 +734,44 @@ def _handle_no_cache_dir(
     help="Don't install package dependencies.",
 )
 
+
+def _handle_dependency_group(
+    option: Option, opt: str, value: str, parser: OptionParser
+) -> None:
+    """
+    Process a value provided for the --group option.
+
+    Splits on the rightmost ":", and validates that the path (if present) ends
+    in `pyproject.toml`. Defaults the path to `pyproject.toml` when one is not given.
+
+    `:` cannot appear in dependency group names, so this is a safe and simple parse.
+
+    This is an optparse.Option callback for the dependency_groups option.
+    """
+    path, sep, groupname = value.rpartition(":")
+    if not sep:
+        path = "pyproject.toml"
+    else:
+        # check for 'pyproject.toml' filenames using pathlib
+        if pathlib.PurePath(path).name != "pyproject.toml":
+            msg = "group paths use 'pyproject.toml' filenames"
+            raise_option_error(parser, option=option, msg=msg)
+
+    parser.values.dependency_groups.append((path, groupname))
+
+
 dependency_groups: Callable[..., Option] = partial(
     Option,
     "--group",
     dest="dependency_groups",
     default=[],
-    action="append",
-    metavar="group",
-    help="Install a named dependency-group from `pyproject.toml` "
-    "in the current directory.",
+    type=str,
+    action="callback",
+    callback=_handle_dependency_group,
+    metavar="[path:]group",
+    help='Install a named dependency-group from a "pyproject.toml" file. '
+    'If a path is given, it must end in "pyproject.toml:". '
+    'Defaults to using "pyproject.toml" in the current directory.',
 )
 
 ignore_requires_python: Callable[..., Option] = partial(

src/pip/_internal/req/req_dependency_group.py

@@ -1,49 +1,74 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, Iterable, Iterator, List, Tuple
 
 from pip._vendor import tomli
-from pip._vendor.dependency_groups import resolve as resolve_dependency_group
+from pip._vendor.dependency_groups import DependencyGroupResolver
 
 from pip._internal.exceptions import InstallationError
 
 
-def parse_dependency_groups(groups: List[str]) -> List[str]:
+def parse_dependency_groups(groups: List[Tuple[str, str]]) -> List[str]:
     """
-    Parse dependency groups data in a way which is sensitive to the `pip` context and
-    raises InstallationErrors if anything goes wrong.
+    Parse dependency groups data as provided via the CLI, in a `[path:]group` syntax.
+
+    Raises InstallationErrors if anything goes wrong.
     """
-    pyproject = _load_pyproject()
-
-    if "dependency-groups" not in pyproject:
-        raise InstallationError(
-            "[dependency-groups] table was missing. Cannot resolve '--group' options."
-        )
-    raw_dependency_groups = pyproject["dependency-groups"]
-    if not isinstance(raw_dependency_groups, dict):
-        raise InstallationError(
-            "[dependency-groups] table was malformed. Cannot resolve '--group' options."
-        )
+    resolvers = _build_resolvers(path for (path, _) in groups)
+    return list(_resolve_all_groups(resolvers, groups))
 
-    try:
-        return list(resolve_dependency_group(raw_dependency_groups, *groups))
-    except (ValueError, TypeError, LookupError) as e:
-        raise InstallationError(f"[dependency-groups] resolution failed: {e}") from e
+
+def _resolve_all_groups(
+    resolvers: Dict[str, DependencyGroupResolver], groups: List[Tuple[str, str]]
+) -> Iterator[str]:
+    """
+    Run all resolution, converting any error from `DependencyGroupResolver` into
+    an InstallationError.
+    """
+    for path, groupname in groups:
+        resolver = resolvers[path]
+        try:
+            yield from (str(req) for req in resolver.resolve(groupname))
+        except (ValueError, TypeError, LookupError) as e:
+            raise InstallationError(
+                f"[dependency-groups] resolution failed for '{groupname}' "
+                f"from '{path}': {e}"
+            ) from e
+
+
+def _build_resolvers(paths: Iterable[str]) -> Dict[str, Any]:
+    resolvers = {}
+    for path in paths:
+        if path in resolvers:
+            continue
+
+        pyproject = _load_pyproject(path)
+        if "dependency-groups" not in pyproject:
+            raise InstallationError(
+                f"[dependency-groups] table was missing from '{path}'. "
+                "Cannot resolve '--group' option."
+            )
+        raw_dependency_groups = pyproject["dependency-groups"]
+        if not isinstance(raw_dependency_groups, dict):
+            raise InstallationError(
+                f"[dependency-groups] table was malformed in {path}. "
+                "Cannot resolve '--group' option."
+            )
+
+        resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
+    return resolvers
 
 
-def _load_pyproject() -> Dict[str, Any]:
+def _load_pyproject(path: str) -> Dict[str, Any]:
     """
-    This helper loads pyproject.toml from the current working directory.
+    This helper loads a pyproject.toml as TOML.
 
-    It does not allow specification of the path to be used and raises an
-    InstallationError if the operation fails.
+    It raises an InstallationError if the operation fails.
     """
     try:
-        with open("pyproject.toml", "rb") as fp:
+        with open(path, "rb") as fp:
             return tomli.load(fp)
     except FileNotFoundError:
-        raise InstallationError(
-            "pyproject.toml not found. Cannot resolve '--group' options."
-        )
+        raise InstallationError(f"{path} not found. Cannot resolve '--group' option.")
     except tomli.TOMLDecodeError as e:
-        raise InstallationError(f"Error parsing pyproject.toml: {e}") from e
+        raise InstallationError(f"Error parsing {path}: {e}") from e
     except OSError as e:
-        raise InstallationError(f"Error reading pyproject.toml: {e}") from e
+        raise InstallationError(f"Error reading {path}: {e}") from e

tests/functional/test_install.py

@@ -333,6 +333,30 @@ def test_install_exit_status_code_when_empty_dependency_group(
     script.pip("install", "--group", "empty")
 
 
+@pytest.mark.parametrize("file_exists", [True, False])
+def test_install_dependency_group_bad_filename_error(
+    script: PipTestEnvironment, file_exists: bool
+) -> None:
+    """
+    Test install exit status code is 2 (usage error) when a dependency group path is
+    specified which isn't a `pyproject.toml`
+    """
+    if file_exists:
+        script.scratch_path.joinpath("not-pyproject.toml").write_text(
+            textwrap.dedent(
+                """
+                [dependency-groups]
+                publish = ["twine"]
+                """
+            )
+        )
+    result = script.pip(
+        "install", "--group", "not-pyproject.toml:publish", expect_error=True
+    )
+    assert "group paths use 'pyproject.toml' filenames" in result.stderr
+    assert result.returncode == 2
+
+
 @pytest.mark.network
 def test_basic_install_from_pypi(script: PipTestEnvironment) -> None:
     """

tests/functional/test_install_reqs.py

@@ -94,10 +94,22 @@ def test_requirements_file(script: PipTestEnvironment) -> None:
 
 
 @pytest.mark.network
-def test_dependency_group(script: PipTestEnvironment) -> None:
+@pytest.mark.parametrize(
+    "path, groupname",
+    [
+        (None, "initools"),
+        ("pyproject.toml", "initools"),
+        ("./pyproject.toml", "initools"),
+        (lambda path: path.absolute(), "initools"),
+    ],
+)
+def test_dependency_group(
+    script: PipTestEnvironment,
+    path: Any,
+    groupname: str,
+) -> None:
     """
     Test installing from a dependency group.
-
     """
     pyproject = script.scratch_path / "pyproject.toml"
     pyproject.write_text(
@@ -111,7 +123,13 @@ def test_dependency_group(script: PipTestEnvironment) -> None:
             """
         )
     )
-    result = script.pip("install", "--group", "initools")
+    if path is None:
+        arg = groupname
+    else:
+        if callable(path):
+            path = path(pyproject)
+        arg = f"{path}:{groupname}"
+    result = script.pip("install", "--group", arg)
     result.did_create(script.site_packages / "INITools-0.2.dist-info")
     result.did_create(script.site_packages / "initools")
     assert result.files_created[script.site_packages / "peppercorn"].dir

tests/unit/test_req_dependency_group.py

@@ -23,7 +23,7 @@ def test_parse_simple_dependency_groups(
     )
     monkeypatch.chdir(tmp_path)
 
-    result = list(parse_dependency_groups(["foo"]))
+    result = list(parse_dependency_groups([("pyproject.toml", "foo")]))
 
     assert len(result) == 1, result
     assert result[0] == "bar"
@@ -45,9 +45,13 @@ def test_parse_cyclic_dependency_groups(
     monkeypatch.chdir(tmp_path)
 
     with pytest.raises(
-        InstallationError, match=r"\[dependency-groups\] resolution failed:"
+        InstallationError,
+        match=(
+            r"\[dependency-groups\] resolution failed for "
+            r"'foo' from 'pyproject\.toml':"
+        ),
     ) as excinfo:
-        parse_dependency_groups(["foo"])
+        parse_dependency_groups([("pyproject.toml", "foo")])
 
     exception = excinfo.value
     assert (
@@ -63,9 +67,10 @@ def test_parse_with_no_dependency_groups_defined(
     monkeypatch.chdir(tmp_path)
 
     with pytest.raises(
-        InstallationError, match=r"\[dependency-groups\] table was missing\."
+        InstallationError,
+        match=(r"\[dependency-groups\] table was missing from 'pyproject\.toml'\."),
     ):
-        parse_dependency_groups(["foo"])
+        parse_dependency_groups([("pyproject.toml", "foo")])
 
 
 def test_parse_with_no_pyproject_file(
@@ -74,7 +79,7 @@ def test_parse_with_no_pyproject_file(
     monkeypatch.chdir(tmp_path)
 
     with pytest.raises(InstallationError, match=r"pyproject\.toml not found\."):
-        parse_dependency_groups(["foo"])
+        parse_dependency_groups([("pyproject.toml", "foo")])
 
 
 def test_parse_with_malformed_pyproject_file(
@@ -92,7 +97,7 @@ def test_parse_with_malformed_pyproject_file(
     monkeypatch.chdir(tmp_path)
 
     with pytest.raises(InstallationError, match=r"Error parsing pyproject\.toml"):
-        parse_dependency_groups(["foo"])
+        parse_dependency_groups([("pyproject.toml", "foo")])
 
 
 def test_parse_gets_unexpected_oserror(
@@ -119,4 +124,4 @@ def epipe_toml_load(*args: Any, **kwargs: Any) -> None:
     )
 
     with pytest.raises(InstallationError, match=r"Error reading pyproject\.toml"):
-        parse_dependency_groups(["foo"])
+        parse_dependency_groups([("pyproject.toml", "foo")])