Remove deprecated pkg_resources usage for setuptools >= 82 compatibility (#473)

* drop pkg resources dependency

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add unit tests for requirements parsing and adjustment functions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update

* Apply suggestions

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Apply suggestions

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Refactor requirement parsing to ensure string conversion of requirements

* Enhance upper bound removal logic in requirement adjustment and add corresponding tests

* Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fix line continuation handling in requirement parsing and add corresponding tests

* Refactor yield_lines to _yield_lines for consistency and update tests accordingly

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

CHANGELOG.md

@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Fixed `compare_version` if runtime error ([#427](https://github.com/Lightning-AI/utilities/pull/427))
+- Remove deprecated `pkg_resources` usage for `setuptools >= 82` compatibility ([#473](https://github.com/Lightning-AI/utilities/pull/473))
 
 
 ---

pyproject.toml

@@ -1,5 +1,6 @@
 [build-system]
 requires = [
+  "packaging",
   "setuptools",
   "wheel",
 ]

setup.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 import glob
 import os
+from collections.abc import Iterator
 from importlib.util import module_from_spec, spec_from_file_location
 
-from pkg_resources import parse_requirements
 from setuptools import find_packages, setup
 
 _PATH_ROOT = os.path.realpath(os.path.dirname(__file__))
@@ -19,10 +19,17 @@ def _load_py_module(fname: str, pkg: str = "lightning_utilities"):
 
 
 about = _load_py_module("__about__.py")
+requirements_module = _load_py_module(os.path.join("install", "requirements.py"))
+
+
+# load basic requirements using the central parser from lightning_utilities.install.requirements
+def _parse_requirements(lines: list[str]) -> Iterator[str]:
+    """Parse requirements from lines using the canonical parser."""
+    return (str(req) for req in requirements_module._parse_requirements(lines))
+
 
-# load basic requirements
 with open(os.path.join(_PATH_REQUIRE, "core.txt")) as fp:
-    requirements = list(map(str, parse_requirements(fp.readlines())))
+    requirements = list(_parse_requirements(fp.readlines()))
 
 
 # make extras as automated loading
@@ -36,8 +43,7 @@ def _requirement_extras(path_req: str = _PATH_REQUIRE) -> dict:
             continue
         name, _ = os.path.splitext(fname)
         with open(fpath) as fp:
-            reqs = parse_requirements(fp.readlines())
-            extras[name] = list(map(str, reqs))
+            extras[name] = list(_parse_requirements(fp.readlines()))
     return extras
 
 

src/lightning_utilities/docs/formatting.py

@@ -86,16 +86,15 @@ def _load_pypi_versions(package_name: str) -> list[str]:
     ['0.9', '0.10', '0.11', '0.12', ...]
 
     """
-    from distutils.version import LooseVersion
-
     import requests
+    from packaging.version import Version
 
     url = f"https://pypi.org/pypi/{package_name}/json"
     data = requests.get(url, timeout=10).json()
     versions = data["releases"].keys()
     # filter all version which include only numbers and dots
     versions = {k for k in versions if re.match(r"^\d+(\.\d+)*$", k)}
-    return sorted(versions, key=LooseVersion)
+    return sorted(versions, key=Version)
 
 
 def _update_link_based_imported_package(link: str, pkg_ver: str, version_digits: Optional[int]) -> str:

src/lightning_utilities/install/requirements.py

@@ -10,11 +10,25 @@
 
 import re
 from collections.abc import Iterable, Iterator
-from distutils.version import LooseVersion
 from pathlib import Path
 from typing import Any, Optional, Union
 
-from pkg_resources import Requirement, yield_lines  # type: ignore[import-untyped]
+from packaging.requirements import Requirement
+from packaging.version import Version
+
+
+def _yield_lines(strs: Union[str, Iterable[str]]) -> Iterator[str]:
+    """Yield non-empty, non-comment lines from a string or iterable of strings.
+
+    Adapted from pkg_resources.yield_lines.
+
+    """
+    if isinstance(strs, str):
+        strs = strs.splitlines()
+    for line in strs:
+        line = line.strip()
+        if line and not line.startswith("#"):
+            yield line
 
 
 class _RequirementWithComment(Requirement):
@@ -77,16 +91,20 @@ def adjust(self, unfreeze: str) -> str:
         if self.strict:
             return f"{out}  {self.strict_string}"
         if unfreeze == "major":
-            for operator, version in self.specs:
-                if operator in ("<", "<="):
-                    major = LooseVersion(version).version[0]
+            for spec in self.specifier:
+                if spec.operator in ("<", "<="):
+                    major = Version(spec.version).major
                     # replace upper bound with major version increased by one
-                    return out.replace(f"{operator}{version}", f"<{int(major) + 1}.0")
+                    return out.replace(f"{spec.operator}{spec.version}", f"<{int(major) + 1}.0")
         elif unfreeze == "all":
-            for operator, version in self.specs:
-                if operator in ("<", "<="):
-                    # drop upper bound
-                    return out.replace(f"{operator}{version},", "")
+            for spec in self.specifier:
+                if spec.operator in ("<", "<="):
+                    # drop upper bound (with or without trailing/leading comma)
+                    upper = f"{spec.operator}{spec.version}"
+                    result = out.replace(f"{upper},", "").replace(f",{upper}", "")
+                    if upper in result:
+                        result = result.replace(upper, "")
+                    return result.strip()
         elif unfreeze != "none":
             raise ValueError(f"Unexpected unfreeze: {unfreeze!r} value.")
         return out
@@ -113,7 +131,7 @@ def _parse_requirements(strs: Union[str, Iterable[str]]) -> Iterator[_Requiremen
         _RequirementWithComment: Parsed requirement objects with preserved comment and pip argument.
 
     """
-    lines = yield_lines(strs)
+    lines = _yield_lines(strs)
     pip_argument = None
     for line in lines:
         # Drop comments -- a hash without a space may be in a URL.
@@ -124,7 +142,7 @@ def _parse_requirements(strs: Union[str, Iterable[str]]) -> Iterator[_Requiremen
             comment = ""
         # If there is a line continuation, drop it, and append the next line.
         if line.endswith("\\"):
-            line = line[:-2].strip()
+            line = line[:-1].strip()
             try:
                 line += next(lines)
             except StopIteration:

tests/unittests/install/__init__.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+
+import pytest
+
+from lightning_utilities.install.requirements import (
+    _parse_requirements,
+    _RequirementWithComment,
+    _yield_lines,
+    load_requirements,
+)
+
+_PATH_ROOT = Path(__file__).parent.parent.parent.parent
+
+
+def test_yield_lines_from_list():
+    assert list(_yield_lines(["foo", "  bar  ", "", "# comment", "baz"])) == ["foo", "bar", "baz"]
+
+
+def test_yield_lines_from_string():
+    assert list(_yield_lines("foo\n  bar  \n\n# comment\nbaz")) == ["foo", "bar", "baz"]
+
+
+def test_yield_lines_empty():
+    assert list(_yield_lines([])) == []
+    assert list(_yield_lines("")) == []
+
+
+def test_requirement_with_comment_attributes():
+    req = _RequirementWithComment("arrow>=1.0", comment="# my comment")
+    assert req.name == "arrow"
+    assert req.comment == "# my comment"
+    assert req.pip_argument is None
+    assert req.strict is False
+
+
+def test_requirement_with_comment_strict():
+    assert _RequirementWithComment("arrow>=1.0", comment="# strict").strict is True
+    assert _RequirementWithComment("arrow>=1.0", comment="# Strict pinning").strict is True
+
+
+def test_requirement_with_comment_pip_argument():
+    req = _RequirementWithComment("arrow>=1.0", pip_argument="--extra-index-url https://x")
+    assert req.pip_argument == "--extra-index-url https://x"
+
+    with pytest.raises(RuntimeError, match="wrong pip argument"):
+        _RequirementWithComment("arrow>=1.0", pip_argument="")
+
+
+def test_adjust_none():
+    assert _RequirementWithComment("arrow<=1.2,>=1.0").adjust("none") == "arrow<=1.2,>=1.0"
+    assert (
+        _RequirementWithComment("arrow<=1.2,>=1.0", comment="# strict").adjust("none") == "arrow<=1.2,>=1.0  # strict"
+    )
+
+
+def test_adjust_all():
+    assert _RequirementWithComment("arrow<=1.2,>=1.0").adjust("all") == "arrow>=1.0"
+    assert _RequirementWithComment("arrow>=1.0,<=1.2").adjust("all") == "arrow>=1.0"
+    assert _RequirementWithComment("arrow<=1.2").adjust("all") == "arrow"
+    assert _RequirementWithComment("arrow<=1.2,>=1.0", comment="# strict").adjust("all") == "arrow<=1.2,>=1.0  # strict"
+    assert _RequirementWithComment("arrow").adjust("all") == "arrow"
+
+
+def test_adjust_major():
+    assert _RequirementWithComment("arrow>=1.2.0, <=1.2.2").adjust("major") == "arrow<2.0,>=1.2.0"
+    assert _RequirementWithComment("lib>=0.5, <=0.9").adjust("major") == "lib<1.0,>=0.5"
+    assert (
+        _RequirementWithComment("arrow>=1.2.0, <=1.2.2", comment="# strict").adjust("major")
+        == "arrow<=1.2.2,>=1.2.0  # strict"
+    )
+    assert _RequirementWithComment("arrow>=1.2.0").adjust("major") == "arrow>=1.2.0"
+
+
+def test_adjust_invalid_unfreeze():
+    with pytest.raises(ValueError, match="Unexpected unfreeze"):
+        _RequirementWithComment("arrow>=1.0").adjust("invalid")
+
+
+def test_parse_requirements_basic():
+    reqs = list(_parse_requirements(["# comment", "", "numpy>=1.0", "pandas<2.0"]))
+    assert [str(r) for r in reqs] == ["numpy>=1.0", "pandas<2.0"]
+
+
+def test_parse_requirements_from_string():
+    reqs = list(_parse_requirements("# comment\n\nnumpy>=1.0\npandas<2.0"))
+    assert [str(r) for r in reqs] == ["numpy>=1.0", "pandas<2.0"]
+
+
+def test_parse_requirements_preserves_comments():
+    reqs = list(_parse_requirements(["arrow>=1.0 # strict"]))
+    assert len(reqs) == 1
+    assert reqs[0].comment == " # strict"
+    assert reqs[0].strict is True
+
+
+def test_parse_requirements_pip_argument():
+    reqs = list(_parse_requirements(["--extra-index-url https://x", "torch>=2.0"]))
+    assert len(reqs) == 1
+    assert reqs[0].pip_argument == "--extra-index-url https://x"
+
+
+def test_parse_requirements_skips():
+    reqs = list(_parse_requirements(["-r other.txt", "pesq @ git+https://github.com/foo/bar", "numpy"]))
+    assert len(reqs) == 1
+    assert reqs[0].name == "numpy"
+
+
+def test_parse_requirements_line_continuation():
+    reqs = list(_parse_requirements(["foo\\", ">=1.0"]))
+    assert len(reqs) == 1
+    assert str(reqs[0]) == "foo>=1.0"
+
+    reqs = list(_parse_requirements(["bar \\", ">=2.0,<3.0"]))
+    assert len(reqs) == 1
+    assert str(reqs[0]) == "bar<3.0,>=2.0"
+
+
+def test_load_requirements_core():
+    path_req = str(_PATH_ROOT / "requirements")
+    reqs = load_requirements(path_req, "core.txt", unfreeze="all")
+    assert len(reqs) > 0
+    # Verify that load_requirements returns a cleaned list of requirement strings
+    assert all(isinstance(r, str) for r in reqs)
+    assert all(r for r in reqs)  # no empty strings
+    assert all(not r.lstrip().startswith("#") for r in reqs)  # no comment lines
+    assert all(r == r.strip() for r in reqs)  # no leading/trailing whitespace
+
+
+def test_load_requirements_nonexistent(tmpdir):
+    with pytest.raises(FileNotFoundError):
+        load_requirements(str(tmpdir), "nonexistent.txt")
+
+
+def test_load_requirements_invalid_unfreeze(tmpdir):
+    with pytest.raises(ValueError, match="unsupported"):
+        load_requirements(str(tmpdir), "x.txt", unfreeze="bad")
+
+
+def test_load_requirements_unfreeze_strategies(tmpdir):
+    req_file = tmpdir / "test.txt"
+    req_file.write("arrow>=1.2.0, <=1.2.2\n")
+
+    assert load_requirements(str(tmpdir), "test.txt", unfreeze="none") == ["arrow<=1.2.2,>=1.2.0"]
+    assert load_requirements(str(tmpdir), "test.txt", unfreeze="major") == ["arrow<2.0,>=1.2.0"]
+    assert load_requirements(str(tmpdir), "test.txt", unfreeze="all") == ["arrow>=1.2.0"]