feat(cli): Support PEP 735 (Dependency Groups)

docs/managing-dependencies.md

@@ -24,17 +24,33 @@ Poetry provides a way to **organize** your dependencies by **groups**.
 The dependencies declared in `project.dependencies` respectively `tool.poetry.dependencies`
 are part of an implicit `main` group. Those dependencies are required by your project during runtime.
 
-Beside the `main` dependencies, you might have dependencies that are only needed to test your project
+Besides the `main` dependencies, you might have dependencies that are only needed to test your project
 or to build the documentation.
 
-To declare a new dependency group, use a `tool.poetry.group.<group>` section
-where `<group>` is the name of your dependency group (for instance, `test`):
+To declare a new dependency group, use a `dependency-groups` section according to PEP 735 or
+a `tool.poetry.group.<group>` section where `<group>` is the name of your dependency group (for instance, `test`):
 
+{{< tabs tabTotal="2" tabID1="group-pep735" tabID2="group-poetry" tabName1="[dependency-groups]" tabName2="[tool.poetry]">}}
+
+{{< tab tabID="group-pep735" >}}
+```toml
+[dependency-groups]
+test = [
+    "pytest (>=6.0.0,<7.0.0)",
+    "pytest-mock",
+]
+```
+{{< /tab >}}
+
+{{< tab tabID="group-poetry" >}}
 ```toml
 [tool.poetry.group.test.dependencies]
 pytest = "^6.0.0"
 pytest-mock = "*"
 ```
+{{< /tab >}}
+{{< /tabs >}}
+
 
 {{% note %}}
 All dependencies **must be compatible with each other** across groups since they will
@@ -60,13 +76,32 @@ A dependency group can be declared as optional. This makes sense when you have
 a group of dependencies that are only required in a particular environment or for
 a specific purpose.
 
+{{< tabs tabTotal="2" tabID1="group-optional-pep735" tabID2="group-optional-poetry" tabName1="[dependency-groups]" tabName2="[tool.poetry]">}}
+
+{{< tab tabID="group-optional-pep735" >}}
+```toml
+[dependency-groups]
+docs = [
+    "mkdocs",
+]
+
+[tool.poetry.group.docs]
+optional = true
+```
+{{< /tab >}}
+
+{{< tab tabID="group-optional-poetry" >}}
 ```toml
 [tool.poetry.group.docs]
 optional = true
 
 [tool.poetry.group.docs.dependencies]
 mkdocs = "*"
 ```
+{{< /tab >}}
+{{< /tabs >}}
+
+
 
 Optional groups can be installed in addition to the **default** dependencies by using the `--with`
 option of the [`install`]({{< relref "cli#install" >}}) command.

src/poetry/console/commands/add.py

@@ -152,14 +152,18 @@ def handle(self) -> int:
         content: dict[str, Any] = self.poetry.file.read()
         project_content = content.get("project", table())
         poetry_content = content.get("tool", {}).get("poetry", table())
+        groups_content = content.get("dependency-groups", {})
         project_name = (
             canonicalize_name(name)
             if (name := project_content.get("name", poetry_content.get("name")))
             else None
         )
 
         use_project_section = False
+        use_groups_section = False
         project_dependency_names = []
+
+        # Run-Time Deps incl. extras
         if group == MAIN_GROUP:
             if (
                 "dependencies" in project_content
@@ -179,22 +183,26 @@ def handle(self) -> int:
                 project_section = array()
 
             poetry_section = poetry_content.get("dependencies", table())
-        else:
-            if "group" not in poetry_content:
-                poetry_content["group"] = table(is_super_table=True)
-
-            groups = poetry_content["group"]
-
-            if group not in groups:
-                groups[group] = table()
-                groups.add(nl())
 
-            this_group = groups[group]
+        # Dependency Groups
+        else:
+            if groups_content or "group" not in poetry_content:
+                use_groups_section = True
+                if not groups_content:
+                    groups_content = table(is_super_table=True)
+                if group not in groups_content:
+                    groups_content[group] = array("[\n]")
 
-            if "dependencies" not in this_group:
-                this_group["dependencies"] = table()
+                project_dependency_names = [
+                    Dependency.create_from_pep_508(dep).name
+                    for dep in groups_content[group]
+                ]
 
-            poetry_section = this_group["dependencies"]
+            poetry_section = (
+                poetry_content.get("group", {})
+                .get(group, {})
+                .get("dependencies", table())
+            )
             project_section = []
 
         existing_packages = self.get_existing_packages_from_input(
@@ -263,17 +271,17 @@ def handle(self) -> int:
                     self.line_error("\nNo changes were applied.")
                     return 1
 
-            if self.option("python"):
-                constraint["python"] = self.option("python")
+            if python := self.option("python"):
+                constraint["python"] = python
 
-            if self.option("platform"):
-                constraint["platform"] = self.option("platform")
+            if platform := self.option("platform"):
+                constraint["platform"] = platform
 
-            if self.option("markers"):
-                constraint["markers"] = self.option("markers")
+            if markers := self.option("markers"):
+                constraint["markers"] = markers
 
-            if self.option("source"):
-                constraint["source"] = self.option("source")
+            if source := self.option("source"):
+                constraint["source"] = source
 
             if len(constraint) == 1 and "version" in constraint:
                 constraint = constraint["version"]
@@ -304,13 +312,16 @@ def handle(self) -> int:
             )
             self.poetry.package.add_dependency(dependency)
 
-            if use_project_section:
+            if use_project_section or use_groups_section:
+                pep_section = (
+                    project_section if use_project_section else groups_content[group]
+                )
                 try:
                     index = project_dependency_names.index(canonical_constraint_name)
                 except ValueError:
-                    project_section.append(dependency.to_pep_508())
+                    pep_section.append(dependency.to_pep_508())
                 else:
-                    project_section[index] = dependency.to_pep_508()
+                    pep_section[index] = dependency.to_pep_508()
 
                 # create a second constraint for tool.poetry.dependencies with keys
                 # that cannot be stored in the project section
@@ -352,13 +363,33 @@ def handle(self) -> int:
                     project_content["optional-dependencies"][optional] = project_section
             elif "dependencies" not in project_content:
                 project_content["dependencies"] = project_section
+
         if poetry_section:
             if "tool" not in content:
                 content["tool"] = table()
             if "poetry" not in content["tool"]:
                 content["tool"]["poetry"] = poetry_content
-            if group == MAIN_GROUP and "dependencies" not in poetry_content:
-                poetry_content["dependencies"] = poetry_section
+            if group == MAIN_GROUP:
+                if "dependencies" not in poetry_content:
+                    poetry_content["dependencies"] = poetry_section
+            else:
+                if "group" not in poetry_content:
+                    poetry_content["group"] = table(is_super_table=True)
+
+                groups = poetry_content["group"]
+
+                if group not in groups:
+                    groups[group] = table()
+                    groups.add(nl())
+
+                if "dependencies" not in groups[group]:
+                    groups[group]["dependencies"] = poetry_section
+
+        if groups_content and group != MAIN_GROUP:
+            if "dependency-groups" not in content:
+                content["dependency-groups"] = table()
+            content["dependency-groups"][group] = groups_content[group]
+
         self.poetry.locker.set_pyproject_data(content)
         self.installer.set_locker(self.poetry.locker)
 

src/poetry/console/commands/remove.py

@@ -63,29 +63,48 @@ def handle(self) -> int:
 
         content: dict[str, Any] = self.poetry.file.read()
         project_content = content.get("project", {})
+        groups_content = content.get("dependency-groups", {})
         poetry_content = content.get("tool", {}).get("poetry", {})
+        poetry_groups_content = poetry_content.get("group", {})
 
         if group is None:
             # remove from all groups
             removed = set()
             group_sections = []
             project_dependencies = project_content.get("dependencies", [])
             poetry_dependencies = poetry_content.get("dependencies", {})
+
             if project_dependencies or poetry_dependencies:
                 group_sections.append(
                     (MAIN_GROUP, project_dependencies, poetry_dependencies)
                 )
+            group_sections.extend(
+                (
+                    group_name,
+                    dependencies,
+                    poetry_groups_content.get(group_name, {}).get("dependencies", {}),
+                )
+                for group_name, dependencies in groups_content.items()
+            )
             group_sections.extend(
                 (group_name, [], group_section.get("dependencies", {}))
-                for group_name, group_section in poetry_content.get("group", {}).items()
+                for group_name, group_section in poetry_groups_content.items()
+                if group_name not in groups_content and group_name != MAIN_GROUP
             )
 
-            for group_name, project_section, poetry_section in group_sections:
+            for group_name, standard_section, poetry_section in group_sections:
                 removed |= self._remove_packages(
-                    packages, project_section, poetry_section, group_name
+                    packages=packages,
+                    standard_section=standard_section,
+                    poetry_section=poetry_section,
+                    group_name=group_name,
                 )
-                if group_name != MAIN_GROUP and not poetry_section:
-                    del poetry_content["group"][group_name]
+                if group_name != MAIN_GROUP:
+                    if not poetry_section and group_name in poetry_groups_content:
+                        del poetry_content["group"][group_name]
+                    if not standard_section and group_name in groups_content:
+                        del groups_content[group_name]
+
         elif group == "dev" and "dev-dependencies" in poetry_content:
             # We need to account for the old `dev-dependencies` section
             removed = self._remove_packages(
@@ -98,18 +117,35 @@ def handle(self) -> int:
             removed = set()
             if "group" in poetry_content:
                 if group in poetry_content["group"]:
-                    removed = self._remove_packages(
-                        packages,
-                        [],
-                        poetry_content["group"][group].get("dependencies", {}),
-                        group,
+                    removed.update(
+                        self._remove_packages(
+                            packages=packages,
+                            standard_section=[],
+                            poetry_section=poetry_content["group"][group].get(
+                                "dependencies", {}
+                            ),
+                            group_name=group,
+                        )
                     )
 
                 if not poetry_content["group"][group]:
                     del poetry_content["group"][group]
+            if group in groups_content:
+                removed.update(
+                    self._remove_packages(
+                        packages=packages,
+                        standard_section=groups_content[group],
+                        poetry_section={},
+                        group_name=group,
+                    )
+                )
+                if not groups_content[group]:
+                    del groups_content[group]
 
         if "group" in poetry_content and not poetry_content["group"]:
             del poetry_content["group"]
+        if "dependency-groups" in content and not content["dependency-groups"]:
+            del content["dependency-groups"]
 
         not_found = set(packages).difference(removed)
         if not_found:
@@ -138,7 +174,7 @@ def handle(self) -> int:
     def _remove_packages(
         self,
         packages: list[str],
-        project_section: list[str],
+        standard_section: list[str],
         poetry_section: dict[str, Any],
         group_name: str,
     ) -> set[str]:
@@ -147,9 +183,9 @@ def _remove_packages(
 
         for package in packages:
             normalized_name = canonicalize_name(package)
-            for requirement in project_section.copy():
+            for requirement in standard_section.copy():
                 if Dependency.create_from_pep_508(requirement).name == normalized_name:
-                    project_section.remove(requirement)
+                    standard_section.remove(requirement)
                     removed.add(package)
             for existing_package in list(poetry_section):
                 if canonicalize_name(existing_package) == normalized_name:

src/poetry/console/commands/self/self_command.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import typing
+
 from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 
 from packaging.utils import canonicalize_name
 from poetry.core.packages.dependency import Dependency
@@ -62,25 +65,35 @@ def activated_groups(self) -> set[NormalizedName]:
 
     def generate_system_pyproject(self) -> None:
         preserved = {}
+        preserved_groups: dict[str, Any] = {}
 
         if self.system_pyproject.exists():
-            content = PyProjectTOML(self.system_pyproject).poetry_config
+            toml_file = PyProjectTOML(self.system_pyproject)
+            content = toml_file.data
 
             for key in {"group", "source"}:
-                if key in content:
-                    preserved[key] = content[key]
+                if key in toml_file.poetry_config:
+                    preserved[key] = toml_file.poetry_config[key]
+
+            if "dependency-groups" in content:
+                preserved_groups = typing.cast(
+                    "dict[str, Any]", content["dependency-groups"]
+                )
 
         package = ProjectPackage(name="poetry-instance", version=__version__)
         package.add_dependency(Dependency(name="poetry", constraint=f"{__version__}"))
 
         package.python_versions = ".".join(str(v) for v in self.env.version_info[:3])
 
-        content = Factory.create_pyproject_from_package(package=package)
+        content = Factory.create_legacy_pyproject_from_package(package=package)
         content["tool"]["poetry"]["package-mode"] = False  # type: ignore[index]
 
         for key in preserved:
             content["tool"]["poetry"][key] = preserved[key]  # type: ignore[index]
 
+        if preserved_groups:
+            content["dependency-groups"] = preserved_groups
+
         pyproject = PyProjectTOML(self.system_pyproject)
         pyproject.file.write(content)
 

src/poetry/factory.py

@@ -220,7 +220,7 @@ def create_package_source(
         )
 
     @classmethod
-    def create_pyproject_from_package(cls, package: Package) -> TOMLDocument:
+    def create_legacy_pyproject_from_package(cls, package: Package) -> TOMLDocument:
         import tomlkit
 
         from poetry.utils.dependency_specification import dependency_to_specification