feat(cli): Support PEP 735 (Dependency Groups)

docs/managing-dependencies.md

@@ -27,14 +27,30 @@ are part of an implicit `main` group. Those dependencies are required by your pr
 Beside the `main` dependencies, you might have dependencies that are only needed to test your project
 or to build the documentation.
 
-To declare a new dependency group, use a `tool.poetry.group.<group>` section
-where `<group>` is the name of your dependency group (for instance, `test`):
+To declare a new dependency group, use a `dependency-groups` section according to PEP 735 or
+a `tool.poetry.group.<group>` section where `<group>` is the name of your dependency group (for instance, `test`):
 
+{{< tabs tabTotal="2" tabID1="group-pep735" tabID2="group-poetry" tabName1="[dependency-groups]" tabName2="[tool.poetry]">}}
+
+{{< tab tabID="group-pep735" >}}
+```toml
+[dependency-groups]
+test = [
+    "pytest (>=6.0.0,<7.0.0)",
+    "pytest-mock",
+]
+```
+{{< /tab >}}
+
+{{< tab tabID="group-poetry" >}}
 ```toml
 [tool.poetry.group.test.dependencies]
 pytest = "^6.0.0"
 pytest-mock = "*"
 ```
+{{< /tab >}}
+{{< /tabs >}}
+
 
 {{% note %}}
 All dependencies **must be compatible with each other** across groups since they will
@@ -60,13 +76,32 @@ A dependency group can be declared as optional. This makes sense when you have
 a group of dependencies that are only required in a particular environment or for
 a specific purpose.
 
+{{< tabs tabTotal="2" tabID1="group-optional-pep735" tabID2="group-optional-poetry" tabName1="[dependency-groups]" tabName2="[tool.poetry]">}}
+
+{{< tab tabID="group-optional-pep735" >}}
+```toml
+[dependency-groups]
+docs = [
+    "mkdocs",
+]
+
+[tool.poetry.group.docs]
+optional = true
+```
+{{< /tab >}}
+
+{{< tab tabID="group-optional-poetry" >}}
 ```toml
 [tool.poetry.group.docs]
 optional = true
 
 [tool.poetry.group.docs.dependencies]
 mkdocs = "*"
 ```
+{{< /tab >}}
+{{< /tabs >}}
+
+
 
 Optional groups can be installed in addition to the **default** dependencies by using the `--with`
 option of the [`install`]({{< relref "cli#install" >}}) command.

pyproject.toml

@@ -4,7 +4,7 @@ version = "2.0.1"
 description = "Python dependency management and packaging made easy."
 requires-python = ">=3.9,<4.0"
 dependencies = [
-    "poetry-core @ git+https://github.com/python-poetry/poetry-core.git",
+    "poetry-core @ git+https://github.com/finswimmer/core.git@poc-pep-735",
     "build (>=1.2.1,<2.0.0)",
     "cachecontrol[filecache] (>=0.14.0,<0.15.0)",
     "cleo (>=2.1.0,<3.0.0)",

src/poetry/console/commands/add.py

@@ -152,14 +152,18 @@ def handle(self) -> int:
         content: dict[str, Any] = self.poetry.file.read()
         project_content = content.get("project", table())
         poetry_content = content.get("tool", {}).get("poetry", table())
+        groups_content = content.get("dependency-groups")
         project_name = (
             canonicalize_name(name)
             if (name := project_content.get("name", poetry_content.get("name")))
             else None
         )
 
         use_project_section = False
+        use_groups_section = False
         project_dependency_names = []
+
+        # Run-Time Deps incl. extras
         if group == MAIN_GROUP:
             if (
                 "dependencies" in project_content
@@ -179,7 +183,21 @@ def handle(self) -> int:
                 project_section = array()
 
             poetry_section = poetry_content.get("dependencies", table())
+
+        # Dependency Groups
         else:
+            if groups_content or "group" not in poetry_content:
+                use_groups_section = True
+                if not groups_content:
+                    groups_content = table(is_super_table=True)
+                if group not in groups_content:
+                    groups_content[group] = array("[\n]")
+
+                project_dependency_names = [
+                    Dependency.create_from_pep_508(dep).name
+                    for dep in groups_content[group]  # type: ignore[union-attr]
+                ]
+
             if "group" not in poetry_content:
                 poetry_content["group"] = table(is_super_table=True)
 
@@ -263,17 +281,17 @@ def handle(self) -> int:
                     self.line_error("\nNo changes were applied.")
                     return 1
 
-            if self.option("python"):
-                constraint["python"] = self.option("python")
+            if python := self.option("python"):
+                constraint["python"] = python
 
-            if self.option("platform"):
-                constraint["platform"] = self.option("platform")
+            if platform := self.option("platform"):
+                constraint["platform"] = platform
 
-            if self.option("markers"):
-                constraint["markers"] = self.option("markers")
+            if markers := self.option("markers"):
+                constraint["markers"] = markers
 
-            if self.option("source"):
-                constraint["source"] = self.option("source")
+            if source := self.option("source"):
+                constraint["source"] = source
 
             if len(constraint) == 1 and "version" in constraint:
                 constraint = constraint["version"]
@@ -304,13 +322,16 @@ def handle(self) -> int:
             )
             self.poetry.package.add_dependency(dependency)
 
-            if use_project_section:
+            if use_project_section or use_groups_section:
+                pep_section = (
+                    project_section if use_project_section else groups_content[group]  # type: ignore[index]
+                )
                 try:
                     index = project_dependency_names.index(canonical_constraint_name)
                 except ValueError:
-                    project_section.append(dependency.to_pep_508())
+                    pep_section.append(dependency.to_pep_508())
                 else:
-                    project_section[index] = dependency.to_pep_508()
+                    pep_section[index] = dependency.to_pep_508()
 
                 # create a second constraint for tool.poetry.dependencies with keys
                 # that cannot be stored in the project section
@@ -352,13 +373,20 @@ def handle(self) -> int:
                     project_content["optional-dependencies"][optional] = project_section
             elif "dependencies" not in project_content:
                 project_content["dependencies"] = project_section
+
         if poetry_section:
             if "tool" not in content:
                 content["tool"] = table()
             if "poetry" not in content["tool"]:
                 content["tool"]["poetry"] = poetry_content
             if group == MAIN_GROUP and "dependencies" not in poetry_content:
                 poetry_content["dependencies"] = poetry_section
+
+        if groups_content and group != MAIN_GROUP:
+            if "dependency-groups" not in content:
+                content["dependency-groups"] = table()
+            content["dependency-groups"][group] = groups_content[group]
+
         self.poetry.locker.set_pyproject_data(content)
         self.installer.set_locker(self.poetry.locker)
 

src/poetry/console/commands/remove.py

@@ -63,6 +63,7 @@ def handle(self) -> int:
 
         content: dict[str, Any] = self.poetry.file.read()
         project_content = content.get("project", {})
+        groups_content = content.get("dependency-groups", {})
         poetry_content = content.get("tool", {}).get("poetry", {})
 
         if group is None:
@@ -71,25 +72,45 @@ def handle(self) -> int:
             group_sections = []
             project_dependencies = project_content.get("dependencies", [])
             poetry_dependencies = poetry_content.get("dependencies", {})
+
             if project_dependencies or poetry_dependencies:
                 group_sections.append(
-                    (MAIN_GROUP, project_dependencies, poetry_dependencies)
+                    (MAIN_GROUP, project_dependencies, poetry_dependencies, [])
                 )
             group_sections.extend(
-                (group_name, [], group_section.get("dependencies", {}))
+                (group_name, [], {}, dependencies)
+                for group_name, dependencies in groups_content.items()
+            )
+            group_sections.extend(
+                (group_name, [], group_section.get("dependencies", {}), [])
                 for group_name, group_section in poetry_content.get("group", {}).items()
             )
 
-            for group_name, project_section, poetry_section in group_sections:
+            for (
+                group_name,
+                project_section,
+                poetry_section,
+                group_dep_section,
+            ) in group_sections:
                 removed |= self._remove_packages(
-                    packages, project_section, poetry_section, group_name
+                    packages=packages,
+                    project_section=project_section,
+                    poetry_section=poetry_section,
+                    group_section=group_dep_section,
+                    group_name=group_name,
                 )
-                if group_name != MAIN_GROUP and not poetry_section:
-                    del poetry_content["group"][group_name]
+                if group_name != MAIN_GROUP:
+                    if not poetry_section and group_name in poetry_content.get(
+                        "group", {}
+                    ):
+                        del poetry_content["group"][group_name]
+                    if not group_dep_section and group_name in groups_content:
+                        del groups_content[group_name]
+
         elif group == "dev" and "dev-dependencies" in poetry_content:
             # We need to account for the old `dev-dependencies` section
             removed = self._remove_packages(
-                packages, [], poetry_content["dev-dependencies"], "dev"
+                packages, [], poetry_content["dev-dependencies"], [], "dev"
             )
 
             if not poetry_content["dev-dependencies"]:
@@ -98,18 +119,37 @@ def handle(self) -> int:
             removed = set()
             if "group" in poetry_content:
                 if group in poetry_content["group"]:
-                    removed = self._remove_packages(
-                        packages,
-                        [],
-                        poetry_content["group"][group].get("dependencies", {}),
-                        group,
+                    removed.update(
+                        self._remove_packages(
+                            packages=packages,
+                            project_section=[],
+                            poetry_section=poetry_content["group"][group].get(
+                                "dependencies", {}
+                            ),
+                            group_section=[],
+                            group_name=group,
+                        )
                     )
 
                 if not poetry_content["group"][group]:
                     del poetry_content["group"][group]
+            if group in groups_content:
+                removed.update(
+                    self._remove_packages(
+                        packages=packages,
+                        project_section=[],
+                        poetry_section={},
+                        group_section=groups_content[group],
+                        group_name=group,
+                    )
+                )
+                if not groups_content[group]:
+                    del groups_content[group]
 
         if "group" in poetry_content and not poetry_content["group"]:
             del poetry_content["group"]
+        if "dependency-groups" in content and not content["dependency-groups"]:
+            del content["dependency-groups"]
 
         not_found = set(packages).difference(removed)
         if not_found:
@@ -140,6 +180,7 @@ def _remove_packages(
         packages: list[str],
         project_section: list[str],
         poetry_section: dict[str, Any],
+        group_section: list[str],
         group_name: str,
     ) -> set[str]:
         removed = set()
@@ -155,6 +196,10 @@ def _remove_packages(
                 if canonicalize_name(existing_package) == normalized_name:
                     del poetry_section[existing_package]
                     removed.add(package)
+            for requirement in group_section.copy():
+                if Dependency.create_from_pep_508(requirement).name == normalized_name:
+                    group_section.remove(requirement)
+                    removed.add(package)
 
         for package in removed:
             group.remove_dependency(package)

src/poetry/console/commands/self/self_command.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import typing
+
 from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 
 from poetry.core.packages.dependency import Dependency
 from poetry.core.packages.project_package import ProjectPackage
@@ -59,25 +62,35 @@ def activated_groups(self) -> set[str]:
 
     def generate_system_pyproject(self) -> None:
         preserved = {}
+        preserved_groups: dict[str, Any] = {}
 
         if self.system_pyproject.exists():
-            content = PyProjectTOML(self.system_pyproject).poetry_config
+            toml_file = PyProjectTOML(self.system_pyproject)
+            content = toml_file.data
 
             for key in {"group", "source"}:
-                if key in content:
-                    preserved[key] = content[key]
+                if key in toml_file.poetry_config:
+                    preserved[key] = toml_file.poetry_config[key]
+
+            if "dependency-groups" in content:
+                preserved_groups = typing.cast(
+                    "dict[str, Any]", content["dependency-groups"]
+                )
 
         package = ProjectPackage(name="poetry-instance", version=__version__)
         package.add_dependency(Dependency(name="poetry", constraint=f"{__version__}"))
 
         package.python_versions = ".".join(str(v) for v in self.env.version_info[:3])
 
-        content = Factory.create_pyproject_from_package(package=package)
+        content = Factory.create_legacy_pyproject_from_package(package=package)
         content["tool"]["poetry"]["package-mode"] = False  # type: ignore[index]
 
         for key in preserved:
             content["tool"]["poetry"][key] = preserved[key]  # type: ignore[index]
 
+        if preserved_groups:
+            content["dependency-groups"] = preserved_groups
+
         pyproject = PyProjectTOML(self.system_pyproject)
         pyproject.file.write(content)
 

src/poetry/factory.py

@@ -220,7 +220,7 @@ def create_package_source(
         )
 
     @classmethod
-    def create_pyproject_from_package(cls, package: Package) -> TOMLDocument:
+    def create_legacy_pyproject_from_package(cls, package: Package) -> TOMLDocument:
         import tomlkit
 
         from poetry.utils.dependency_specification import dependency_to_specification