Merge pull request #10249 from sbidoul/list-freeze-pep610-sbi

Support PEP 610 editables in pip freeze and pip list

Author: sbidoul

docs/html/cli/pip_list.rst

@@ -139,3 +139,93 @@ Examples
          docopt==0.6.2
          idlex==1.13
          jedi==0.9.0
+
+#. List packages installed in editable mode
+
+When some packages are installed in editable mode, ``pip list`` outputs an
+additional column that shows the directory where the editable project is
+located (i.e. the directory that contains the ``pyproject.toml`` or
+``setup.py`` file).
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list
+         Package          Version  Editable project location
+         ---------------- -------- -------------------------------------
+         pip              21.2.4
+         pip-test-package 0.1.1    /home/you/.venv/src/pip-test-package
+         setuptools       57.4.0
+         wheel            0.36.2
+
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list
+         Package          Version  Editable project location
+         ---------------- -------- ----------------------------------------
+         pip              21.2.4
+         pip-test-package 0.1.1    C:\Users\You\.venv\src\pip-test-package
+         setuptools       57.4.0
+         wheel            0.36.2
+
+The json format outputs an additional ``editable_project_location`` field.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list --format=json | python -m json.tool
+         [
+           {
+             "name": "pip",
+             "version": "21.2.4",
+           },
+           {
+             "name": "pip-test-package",
+             "version": "0.1.1",
+             "editable_project_location": "/home/you/.venv/src/pip-test-package"
+           },
+           {
+             "name": "setuptools",
+             "version": "57.4.0"
+           },
+           {
+             "name": "wheel",
+             "version": "0.36.2"
+           }
+         ]
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list --format=json | py -m json.tool
+         [
+           {
+             "name": "pip",
+             "version": "21.2.4",
+           },
+           {
+             "name": "pip-test-package",
+             "version": "0.1.1",
+             "editable_project_location": "C:\Users\You\.venv\src\pip-test-package"
+           },
+           {
+             "name": "setuptools",
+             "version": "57.4.0"
+           },
+           {
+             "name": "wheel",
+             "version": "0.36.2"
+           }
+         ]
+
+.. note::
+
+   Contrary to the ``freeze``  comand, ``pip list --format=freeze`` will not
+   report editable install information, but the version of the package at the
+   time it was installed.

news/10249.feature.rst

@@ -0,0 +1,4 @@
+Support `PEP 610 <https://www.python.org/dev/peps/pep-0610/>`_ to detect
+editable installs in ``pip freeze`` and  ``pip list``. The ``pip list`` column output
+has a new ``Editable project location`` column, and the JSON output has a new
+``editable_project_location`` field.

src/pip/_internal/commands/list.py

@@ -14,7 +14,8 @@
 from pip._internal.metadata import BaseDistribution, get_environment
 from pip._internal.models.selection_prefs import SelectionPreferences
 from pip._internal.network.session import PipSession
-from pip._internal.utils.misc import stdlib_pkgs, tabulate, write_output
+from pip._internal.utils.compat import stdlib_pkgs
+from pip._internal.utils.misc import tabulate, write_output
 from pip._internal.utils.parallel import map_multithread
 
 if TYPE_CHECKING:
@@ -302,19 +303,22 @@ def format_for_columns(
     Convert the package data into something usable
     by output_package_listing_columns.
     """
+    header = ["Package", "Version"]
+
     running_outdated = options.outdated
-    # Adjust the header for the `pip list --outdated` case.
     if running_outdated:
-        header = ["Package", "Version", "Latest", "Type"]
-    else:
-        header = ["Package", "Version"]
+        header.extend(["Latest", "Type"])
 
-    data = []
-    if options.verbose >= 1 or any(x.editable for x in pkgs):
+    has_editables = any(x.editable for x in pkgs)
+    if has_editables:
+        header.append("Editable project location")
+
+    if options.verbose >= 1:
         header.append("Location")
     if options.verbose >= 1:
         header.append("Installer")
 
+    data = []
     for proj in pkgs:
         # if we're working on the 'outdated' list, separate out the
         # latest_version and type
@@ -324,7 +328,10 @@ def format_for_columns(
             row.append(str(proj.latest_version))
             row.append(proj.latest_filetype)
 
-        if options.verbose >= 1 or proj.editable:
+        if has_editables:
+            row.append(proj.editable_project_location or "")
+
+        if options.verbose >= 1:
             row.append(proj.location or "")
         if options.verbose >= 1:
             row.append(proj.installer)
@@ -347,5 +354,8 @@ def format_for_json(packages: "_ProcessedDists", options: Values) -> str:
         if options.outdated:
             info["latest_version"] = str(dist.latest_version)
             info["latest_filetype"] = dist.latest_filetype
+        editable_project_location = dist.editable_project_location
+        if editable_project_location:
+            info["editable_project_location"] = editable_project_location
         data.append(info)
     return json.dumps(data)

src/pip/_internal/metadata/base.py

@@ -24,7 +24,9 @@
     DirectUrl,
     DirectUrlValidationError,
 )
-from pip._internal.utils.misc import stdlib_pkgs  # TODO: Move definition here.
+from pip._internal.utils.compat import stdlib_pkgs  # TODO: Move definition here.
+from pip._internal.utils.egg_link import egg_link_path_from_sys_path
+from pip._internal.utils.urls import url_to_path
 
 if TYPE_CHECKING:
     from typing import Protocol
@@ -73,6 +75,28 @@ def location(self) -> Optional[str]:
         """
         raise NotImplementedError()
 
+    @property
+    def editable_project_location(self) -> Optional[str]:
+        """The project location for editable distributions.
+
+        This is the directory where pyproject.toml or setup.py is located.
+        None if the distribution is not installed in editable mode.
+        """
+        # TODO: this property is relatively costly to compute, memoize it ?
+        direct_url = self.direct_url
+        if direct_url:
+            if direct_url.is_local_editable():
+                return url_to_path(direct_url.url)
+        else:
+            # Search for an .egg-link file by walking sys.path, as it was
+            # done before by dist_is_editable().
+            egg_link_path = egg_link_path_from_sys_path(self.raw_name)
+            if egg_link_path:
+                # TODO: get project location from second line of egg_link file
+                #       (https://github.com/pypa/pip/issues/10243)
+                return self.location
+        return None
+
     @property
     def info_directory(self) -> Optional[str]:
         """Location of the .[egg|dist]-info directory.
@@ -129,7 +153,7 @@ def installer(self) -> str:
 
     @property
     def editable(self) -> bool:
-        raise NotImplementedError()
+        return bool(self.editable_project_location)
 
     @property
     def local(self) -> bool:

src/pip/_internal/metadata/pkg_resources.py

@@ -69,10 +69,6 @@ def version(self) -> DistributionVersion:
     def installer(self) -> str:
         return get_installer(self._dist)
 
-    @property
-    def editable(self) -> bool:
-        return misc.dist_is_editable(self._dist)
-
     @property
     def local(self) -> bool:
         return misc.dist_is_local(self._dist)

src/pip/_internal/models/direct_url.py

@@ -215,3 +215,6 @@ def from_json(cls, s: str) -> "DirectUrl":
 
     def to_json(self) -> str:
         return json.dumps(self.to_dict(), sort_keys=True)
+
+    def is_local_editable(self) -> bool:
+        return isinstance(self.info, DirInfo) and self.info.editable

src/pip/_internal/operations/freeze.py

@@ -169,16 +169,9 @@ def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
     """
     if not dist.editable:
         return _EditableInfo(requirement=None, editable=False, comments=[])
-    if dist.location is None:
-        display = _format_as_name_version(dist)
-        logger.warning("Editable requirement not found on disk: %s", display)
-        return _EditableInfo(
-            requirement=None,
-            editable=True,
-            comments=[f"# Editable install not found ({display})"],
-        )
-
-    location = os.path.normcase(os.path.abspath(dist.location))
+    editable_project_location = dist.editable_project_location
+    assert editable_project_location
+    location = os.path.normcase(os.path.abspath(editable_project_location))
 
     from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs
 

src/pip/_internal/req/req_uninstall.py

@@ -12,12 +12,12 @@
 from pip._internal.exceptions import UninstallationError
 from pip._internal.locations import get_bin_prefix, get_bin_user
 from pip._internal.utils.compat import WINDOWS
+from pip._internal.utils.egg_link import egg_link_path_from_location
 from pip._internal.utils.logging import getLogger, indent_log
 from pip._internal.utils.misc import (
     ask,
     dist_in_usersite,
     dist_is_local,
-    egg_link_path,
     is_local,
     normalize_path,
     renames,
@@ -459,7 +459,7 @@ def from_dist(cls, dist: Distribution) -> "UninstallPathSet":
             return cls(dist)
 
         paths_to_remove = cls(dist)
-        develop_egg_link = egg_link_path(dist)
+        develop_egg_link = egg_link_path_from_location(dist.project_name)
         develop_egg_link_egg_info = "{}.egg-info".format(
             pkg_resources.to_filename(dist.project_name)
         )

src/pip/_internal/utils/egg_link.py

@@ -0,0 +1,75 @@
+# The following comment should be removed at some point in the future.
+# mypy: strict-optional=False
+
+import os
+import re
+import sys
+from typing import Optional
+
+from pip._internal.locations import site_packages, user_site
+from pip._internal.utils.virtualenv import (
+    running_under_virtualenv,
+    virtualenv_no_global,
+)
+
+__all__ = [
+    "egg_link_path_from_sys_path",
+    "egg_link_path_from_location",
+]
+
+
+def _egg_link_name(raw_name: str) -> str:
+    """
+    Convert a Name metadata value to a .egg-link name, by applying
+    the same substitution as pkg_resources's safe_name function.
+    Note: we cannot use canonicalize_name because it has a different logic.
+    """
+    return re.sub("[^A-Za-z0-9.]+", "-", raw_name) + ".egg-link"
+
+
+def egg_link_path_from_sys_path(raw_name: str) -> Optional[str]:
+    """
+    Look for a .egg-link file for project name, by walking sys.path.
+    """
+    egg_link_name = _egg_link_name(raw_name)
+    for path_item in sys.path:
+        egg_link = os.path.join(path_item, egg_link_name)
+        if os.path.isfile(egg_link):
+            return egg_link
+    return None
+
+
+def egg_link_path_from_location(raw_name: str) -> Optional[str]:
+    """
+    Return the path for the .egg-link file if it exists, otherwise, None.
+
+    There's 3 scenarios:
+    1) not in a virtualenv
+       try to find in site.USER_SITE, then site_packages
+    2) in a no-global virtualenv
+       try to find in site_packages
+    3) in a yes-global virtualenv
+       try to find in site_packages, then site.USER_SITE
+       (don't look in global location)
+
+    For #1 and #3, there could be odd cases, where there's an egg-link in 2
+    locations.
+
+    This method will just return the first one found.
+    """
+    sites = []
+    if running_under_virtualenv():
+        sites.append(site_packages)
+        if not virtualenv_no_global() and user_site:
+            sites.append(user_site)
+    else:
+        if user_site:
+            sites.append(user_site)
+        sites.append(site_packages)
+
+    egg_link_name = _egg_link_name(raw_name)
+    for site in sites:
+        egglink = os.path.join(site, egg_link_name)
+        if os.path.isfile(egglink):
+            return egglink
+    return None