Extract all dependencies from poetry show to divide into direct & transitive

Author: ArBridgeman

exasol/toolbox/util/dependencies.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+import tomlkit
+from pydantic import (
+    BaseModel,
+    model_validator,
+)
+from tomlkit import TOMLDocument
+
+
+class PoetryGroup(BaseModel):
+    name: str
+    toml_section: str
+
+    class Config:
+        frozen = True
+
+
+class Package(BaseModel):
+    name: str
+    version: str
+
+    class Config:
+        frozen = True
+
+    @property
+    def normalized_name(self) -> str:
+        return self.name.lower().replace("_", "-")
+
+
+class PoetryDependency(Package):
+    name: str
+    version: str
+    group: PoetryGroup | None
+
+
+class PoetryToml(BaseModel):
+    working_directory: Path
+
+    @model_validator(mode="before")
+    def read_content(cls, values):
+        file_path = values["working_directory"] / "pyproject.toml"
+        if not file_path.exists():
+            raise ValueError(f"File not found: {file_path}")
+
+        try:
+            text = file_path.read_text()
+            cls._content: TOMLDocument = tomlkit.loads(text)
+        except Exception as e:
+            raise ValueError(f"Error reading file: {str(e)}")
+        return values
+
+    def get_section_dict(self, section: str) -> dict | None:
+        current = self._content.copy()
+        for section in section.split("."):
+            if section not in current:
+                return None
+            current = current[section]
+        return current
+
+    @property
+    def groups(self) -> tuple[PoetryGroup, ...]:
+        groups = []
+
+        main_key = "project.dependencies"
+        if self.get_section_dict(main_key):
+            groups.append(PoetryGroup(name="main", toml_section=main_key))
+
+        main_dynamic_key = "tool.poetry.dependencies"
+        if self.get_section_dict(main_dynamic_key):
+            groups.append(PoetryGroup(name="main", toml_section=main_dynamic_key))
+
+        group_key = "tool.poetry.group"
+        if group_dict := self.get_section_dict(group_key):
+            for group, content in group_dict.items():
+                if "dependencies" in content:
+                    groups.append(
+                        PoetryGroup(
+                            name=group,
+                            toml_section=f"{group_key}.{group}.dependencies",
+                        )
+                    )
+        return tuple(groups)
+
+
+class PoetryDependencies(BaseModel):
+    groups: tuple[PoetryGroup, ...]
+    working_directory: Path
+
+    @staticmethod
+    def _extract_from_line(line: str, group: PoetryGroup | None) -> PoetryDependency:
+        pattern = r"\s+(\d+(?:\.\d+)*)\s+"
+        match = re.split(pattern, line)
+        return PoetryDependency(name=match[0], version=match[1], group=group)
+
+    def _extract_from_poetry_show(
+        self, output_text: str, group: PoetryGroup | None
+    ) -> list[PoetryDependency]:
+        return [
+            self._extract_from_line(line, group=group)
+            for line in output_text.splitlines()
+        ]
+
+    @property
+    def direct_dependencies(self) -> dict[str, list[PoetryDependency]]:
+        dependencies = {}
+        for group in self.groups:
+            command = ("poetry", "show", "--top-level", f"--only={group.name}")
+            output = subprocess.run(
+                command, capture_output=True, text=True, cwd=self.working_directory
+            )
+            result = self._extract_from_poetry_show(
+                output_text=output.stdout, group=group
+            )
+            dependencies[group.name] = result
+        return dependencies
+
+    @property
+    def all_dependencies(self) -> dict[str, list[PoetryDependency]]:
+        command = ("poetry", "show")
+        output = subprocess.run(
+            command, capture_output=True, text=True, cwd=self.working_directory
+        )
+
+        direct_dependencies = self.direct_dependencies.copy()
+        transitive_dependencies = []
+        names_direct_dependencies = {
+            dep.name
+            for group_list in direct_dependencies.values()
+            for dep in group_list
+        }
+        for line in output.stdout.splitlines():
+            dep = self._extract_from_line(line=line, group=None)
+            if dep.name not in names_direct_dependencies:
+                transitive_dependencies.append(dep)
+
+        return direct_dependencies | {"transitive": transitive_dependencies}

exasol/toolbox/util/dependencies/__init__.py

@@ -22,7 +22,6 @@ def test_dependencies():
         autoimport = "^1.4.0"
     """
 
-
     actual = _dependencies(toml)
     assert actual == {"project": ["pytest", "python"], "dev": ["autoimport"]}
 

exasol/toolbox/util/dependencies/poetry_dependencies.py

@@ -0,0 +1,132 @@
+import subprocess
+
+import pytest
+from toolbox.util.dependencies.poetry_dependencies import (
+    PoetryDependencies,
+    PoetryDependency,
+    PoetryGroup,
+    PoetryToml,
+)
+
+MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies")
+DEV_GROUP = PoetryGroup(name="dev", toml_section="tool.poetry.group.dev.dependencies")
+ANALYSIS_GROUP = PoetryGroup(
+    name="analysis", toml_section="tool.poetry.group.analysis.dependencies"
+)
+
+DIRECT_DEPENDENCIES = {
+    MAIN_GROUP.name: [
+        PoetryDependency(name="numpy", version="2.2.6", group=MAIN_GROUP),
+        PoetryDependency(name="pylint", version="3.3.7", group=MAIN_GROUP),
+    ],
+    DEV_GROUP.name: [PoetryDependency(name="isort", version="6.0.1", group=DEV_GROUP)],
+    ANALYSIS_GROUP.name: [
+        PoetryDependency(name="black", version="25.1.0", group=ANALYSIS_GROUP)
+    ],
+}
+
+
+@pytest.fixture(scope="module")
+def cwd(tmp_path_factory):
+    return tmp_path_factory.mktemp("test")
+
+
+@pytest.fixture(scope="module")
+def project_name():
+    return "project"
+
+
+@pytest.fixture(scope="module")
+def project_path(cwd, project_name):
+    return cwd / project_name
+
+
+@pytest.fixture(scope="module")
+def create_poetry_project(cwd, project_name, project_path):
+    subprocess.run(["poetry", "new", project_name], cwd=cwd)
+    subprocess.run(["poetry", "add", "numpy"], cwd=project_path)
+    subprocess.run(["poetry", "add", "pylint"], cwd=project_path)
+    subprocess.run(["poetry", "add", "--group", "dev", "isort"], cwd=project_path)
+    subprocess.run(["poetry", "add", "--group", "analysis", "black"], cwd=project_path)
+
+
+@pytest.fixture(scope="module")
+def pyproject_toml(project_path, create_poetry_project):
+    return PoetryToml(working_directory=project_path)
+
+
+class TestPackage:
+    @staticmethod
+    @pytest.mark.parametrize(
+        "name,expected",
+        [
+            ("numpy", "numpy"),
+            ("sphinxcontrib-applehelp", "sphinxcontrib-applehelp"),
+            ("Imaginary_package", "imaginary-package"),
+        ],
+    )
+    def test_normalized_name(name, expected):
+        dep = PoetryDependency(name=name, version="0.1.0", group=None)
+        assert dep.normalized_name == expected
+
+
+class TestPoetryToml:
+    @staticmethod
+    def test_get_section_dict_exists(pyproject_toml):
+        result = pyproject_toml.get_section_dict("project")
+        assert result is not None
+
+    @staticmethod
+    def test_get_section_dict_does_not_exist(pyproject_toml):
+        result = pyproject_toml.get_section_dict("test")
+        assert result is None
+
+    @staticmethod
+    def test_groups(pyproject_toml):
+        assert pyproject_toml.groups == (MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP)
+
+
+class TestPoetryDependencies:
+    @staticmethod
+    @pytest.mark.parametrize(
+        "line,expected",
+        [
+            (
+                "coverage                      7.8.0     Code coverage measurement for Python",
+                PoetryDependency(name="coverage", version="7.8.0", group=MAIN_GROUP),
+            ),
+            (
+                "furo                          2024.8.6  A clean customisable Sphinx documentation theme.",
+                PoetryDependency(name="furo", version="2024.8.6", group=MAIN_GROUP),
+            ),
+            (
+                "import-linter                 2.3       Enforces rules for the imports within and between Python packages.",
+                PoetryDependency(name="import-linter", version="2.3", group=MAIN_GROUP),
+            ),
+        ],
+    )
+    def test_extract_from_line(line, expected):
+        result = PoetryDependencies._extract_from_line(line=line, group=MAIN_GROUP)
+        assert result == expected
+
+    @staticmethod
+    def test_direct_dependencies(create_poetry_project, project_path):
+        poetry_dep = PoetryDependencies(
+            groups=(MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP),
+            working_directory=project_path,
+        )
+        assert poetry_dep.direct_dependencies == DIRECT_DEPENDENCIES
+
+    @staticmethod
+    def test_all_dependencies(create_poetry_project, project_path):
+        # set_direct_deps = set(DIRECT_DEPENDENCIES)
+
+        poetry_dep = PoetryDependencies(
+            groups=(MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP),
+            working_directory=project_path,
+        )
+        result = poetry_dep.all_dependencies
+
+        transitive = result.pop("transitive")
+        assert len(transitive) > 0
+        assert result == DIRECT_DEPENDENCIES