Merge pull request #46 from DavidCEllis/cache_runs

Cache details of Python installs that are queried

Author: DavidCEllis

.gitignore

@@ -117,6 +117,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
+.venv_*/
 env/
 venv/
 ENV/

src/ducktools/pythonfinder/__init__.py

@@ -32,7 +32,7 @@
 
 import sys
 from ._version import __version__
-from .shared import PythonInstall
+from .shared import PythonInstall, DetailFinder
 
 
 if sys.platform == "win32":
@@ -43,9 +43,10 @@
     from .linux import get_python_installs
 
 
-def list_python_installs(*, query_executables=True):
+def list_python_installs(*, query_executables: bool = True, finder: "DetailFinder | None" = None):
+    finder = DetailFinder() if finder is None else finder
     return sorted(
-        get_python_installs(query_executables=query_executables),
+        get_python_installs(query_executables=query_executables, finder=finder),
         reverse=True,
         key=lambda x: (x.version[3], *x.version[:3], x.version[4])
     )

src/ducktools/pythonfinder/details_script.py

@@ -29,11 +29,13 @@
 FULL_PY_VER_RE = r"(?P<major>\d+)\.(?P<minor>\d+)\.?(?P<micro>\d*)-?(?P<releaselevel>a|b|c|rc)?(?P<serial>\d*)?"
 
 
-def version_str_to_tuple(version):
+def version_str_to_tuple(version: str):
     # Needed to parse GraalPy versions only available as strings
     import re
 
     parsed_version = re.fullmatch(FULL_PY_VER_RE, version)
+    if parsed_version is None:
+        raise ValueError(f"'version' must be a valid Python version string, not {version!r}")
 
     major, minor, micro, releaselevel, serial = parsed_version.groups()
 
@@ -70,11 +72,11 @@ def get_details():
             # Special case GraalPy as it erroneously reports the CPython target
             # instead of the Graal versiion
             try:
-                ver = __graalpython__.get_graalvm_version()
+                ver = __graalpython__.get_graalvm_version()  # type: ignore
                 metadata = {
                     "graalpy_version": version_str_to_tuple(ver)
                 }
-            except NameError:
+            except (NameError, ValueError):
                 metadata = {"{}_version".format(implementation): sys.implementation.version}
         elif implementation != "cpython":  # pragma: no cover
             metadata = {"{}_version".format(implementation): sys.implementation.version}

src/ducktools/pythonfinder/linux/__init__.py

@@ -27,11 +27,17 @@
 import itertools
 from _collections_abc import Iterator
 
-from ..shared import PythonInstall, get_folder_pythons, get_uv_pythons, get_uv_python_path
+from ..shared import (
+    DetailFinder,
+    PythonInstall,
+    get_folder_pythons,
+    get_uv_pythons,
+    get_uv_python_path
+)
 from .pyenv_search import get_pyenv_pythons, get_pyenv_root
 
 
-def get_path_pythons() -> Iterator[PythonInstall]:
+def get_path_pythons(*, finder: DetailFinder | None = None) -> Iterator[PythonInstall]:
     exe_names = set()
 
     path_folders = os.environ.get("PATH", "").split(":")
@@ -40,6 +46,8 @@ def get_path_pythons() -> Iterator[PythonInstall]:
 
     excluded_folders = [pyenv_root, uv_root]
 
+    finder = DetailFinder if finder is None else finder
+
     for fld in path_folders:
         # Don't retrieve pyenv installs
         skip_folder = False
@@ -54,7 +62,7 @@ def get_path_pythons() -> Iterator[PythonInstall]:
         if not os.path.exists(fld):
             continue
 
-        for install in get_folder_pythons(fld):
+        for install in get_folder_pythons(fld, finder=finder):
             name = os.path.basename(install.executable)
             if name in exe_names:
                 install.shadowed = True
@@ -63,17 +71,24 @@ def get_path_pythons() -> Iterator[PythonInstall]:
             yield install
 
 
-def get_python_installs(*, query_executables=True):
+def get_python_installs(
+    *,
+    query_executables: bool = True,
+    finder: DetailFinder | None = None,
+) -> Iterator[PythonInstall]:
     listed_pythons = set()
 
+    finder = DetailFinder() if finder is None else finder
+
     chain_commands = [
-        get_pyenv_pythons(query_executables=query_executables),
-        get_uv_pythons(query_executables=query_executables),
+        get_pyenv_pythons(query_executables=query_executables, finder=finder),
+        get_uv_pythons(query_executables=query_executables, finder=finder),
     ]
     if query_executables:
-        chain_commands.append(get_path_pythons())
+        chain_commands.append(get_path_pythons(finder=finder))
 
-    for py in itertools.chain.from_iterable(chain_commands):
-        if py.executable not in listed_pythons:
-            yield py
-            listed_pythons.add(py.executable)
+    with finder:
+        for py in itertools.chain.from_iterable(chain_commands):
+            if py.executable not in listed_pythons:
+                yield py
+                listed_pythons.add(py.executable)

src/ducktools/pythonfinder/linux/pyenv_search.py

@@ -22,7 +22,6 @@
 # SOFTWARE.
 from __future__ import annotations
 
-
 """
 Discover python installs that have been created with pyenv
 """
@@ -33,10 +32,11 @@
 
 from ducktools.lazyimporter import LazyImporter, FromImport, ModuleImport
 
-from ..shared import PythonInstall, get_install_details, FULL_PY_VER_RE, version_str_to_tuple
+from ..shared import PythonInstall, DetailFinder, FULL_PY_VER_RE, INSTALLER_CACHE_PATH, version_str_to_tuple
 
 _laz = LazyImporter(
     [
+        ModuleImport("json"),
         ModuleImport("re"),
         FromImport("subprocess", "run"),
     ]
@@ -52,11 +52,24 @@ def get_pyenv_root() -> str | None:
     pyenv_root = os.environ.get("PYENV_ROOT")
     if not pyenv_root:
         try:
-            output = _laz.run(["pyenv", "root"], capture_output=True, text=True)
-        except FileNotFoundError:
-            return None
+            with open(INSTALLER_CACHE_PATH) as f:
+                installer_cache = _laz.json.load(f)
+        except (FileNotFoundError, _laz.json.JSONDecodeError):
+            installer_cache = {}
+
+        pyenv_root = installer_cache.get("pyenv")
+        if pyenv_root is None or not os.path.exists(pyenv_root):
+            try:
+                output = _laz.run(["pyenv", "root"], capture_output=True, text=True)
+            except FileNotFoundError:
+                return None
+
+            pyenv_root = output.stdout.strip()
 
-        pyenv_root = output.stdout.strip()
+            installer_cache["pyenv"] = pyenv_root
+            os.makedirs(os.path.dirname(INSTALLER_CACHE_PATH), exist_ok=True)
+            with open(INSTALLER_CACHE_PATH, 'w') as f:
+                _laz.json.dump(installer_cache, f)
 
     return pyenv_root
 
@@ -65,6 +78,7 @@ def get_pyenv_pythons(
     versions_folder: str | os.PathLike | None = None,
     *,
     query_executables: bool = True,
+    finder: DetailFinder = None,
 ) -> Iterator[PythonInstall]:
     if versions_folder is None:
         if pyenv_root := get_pyenv_root():
@@ -77,26 +91,32 @@ def get_pyenv_pythons(
     # This can lead to much faster returns by potentially yielding
     # the required python version before checking pypy/graalpy/micropython
 
-    for p in sorted(os.scandir(versions_folder), key=lambda x: x.path):
-        executable = os.path.join(p.path, "bin/python")
-
-        if os.path.exists(executable):
-            if p.name.endswith("t"):
-                freethreaded = True
-                version = p.name[:-1]
-            else:
-                freethreaded = False
-                version = p.name
-            if _laz.re.fullmatch(FULL_PY_VER_RE, version):
-                version_tuple = version_str_to_tuple(version)
-                metadata = {}
-                if version_tuple >= (3, 13):
-                    metadata["freethreaded"] = freethreaded
-                yield PythonInstall(
-                    version=version_tuple,
-                    executable=executable,
-                    metadata=metadata,
-                    managed_by="pyenv",
-                )
-            elif query_executables and (install := get_install_details(executable, managed_by="pyenv")):
-                yield install
+    finder = DetailFinder() if finder is None else finder
+
+    with finder:
+        for p in sorted(os.scandir(versions_folder), key=lambda x: x.path):
+            executable = os.path.join(p.path, "bin/python")
+
+            if os.path.exists(executable):
+                if p.name.endswith("t"):
+                    freethreaded = True
+                    version = p.name[:-1]
+                else:
+                    freethreaded = False
+                    version = p.name
+                if _laz.re.fullmatch(FULL_PY_VER_RE, version):
+                    version_tuple = version_str_to_tuple(version)
+                    metadata = {}
+                    if version_tuple >= (3, 13):
+                        metadata["freethreaded"] = freethreaded
+                    yield PythonInstall(
+                        version=version_tuple,
+                        executable=executable,
+                        metadata=metadata,
+                        managed_by="pyenv",
+                    )
+                elif (
+                    query_executables
+                    and (install := finder.get_install_details(executable, managed_by="pyenv"))
+                ):
+                    yield install