gh-69605: Hardcode some stdlib submodules in PyREPL module completion (os.path, collections.abc...) (GH-138268)

Co-authored-by: Łukasz Langa <[email protected]>

Author: loic-simon

Lib/_pyrepl/_module_completer.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+import importlib
+import os
 import pkgutil
 import sys
 import token
 import tokenize
+from importlib.machinery import FileFinder
 from io import StringIO
 from contextlib import contextmanager
 from dataclasses import dataclass
@@ -16,6 +19,15 @@
     from typing import Any, Iterable, Iterator, Mapping
 
 
+HARDCODED_SUBMODULES = {
+    # Standard library submodules that are not detected by pkgutil.iter_modules
+    # but can be imported, so should be proposed in completion
+    "collections": ["abc"],
+    "os": ["path"],
+    "xml.parsers.expat": ["errors", "model"],
+}
+
+
 def make_default_module_completer() -> ModuleCompleter:
     # Inside pyrepl, __package__ is set to None by default
     return ModuleCompleter(namespace={'__package__': None})
@@ -41,6 +53,7 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None:
         self.namespace = namespace or {}
         self._global_cache: list[pkgutil.ModuleInfo] = []
         self._curr_sys_path: list[str] = sys.path[:]
+        self._stdlib_path = os.path.dirname(importlib.__path__[0])
 
     def get_completions(self, line: str) -> list[str] | None:
         """Return the next possible import completions for 'line'."""
@@ -95,12 +108,26 @@ def _find_modules(self, path: str, prefix: str) -> list[str]:
                 return []
 
         modules: Iterable[pkgutil.ModuleInfo] = self.global_cache
+        is_stdlib_import: bool | None = None
         for segment in path.split('.'):
             modules = [mod_info for mod_info in modules
                        if mod_info.ispkg and mod_info.name == segment]
+            if is_stdlib_import is None:
+                # Top-level import decide if we import from stdlib or not
+                is_stdlib_import = all(
+                    self._is_stdlib_module(mod_info) for mod_info in modules
+                )
             modules = self.iter_submodules(modules)
-        return [module.name for module in modules
-                if self.is_suggestion_match(module.name, prefix)]
+
+        module_names = [module.name for module in modules]
+        if is_stdlib_import:
+            module_names.extend(HARDCODED_SUBMODULES.get(path, ()))
+        return [module_name for module_name in module_names
+                if self.is_suggestion_match(module_name, prefix)]
+
+    def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool:
+        return (isinstance(module_info.module_finder, FileFinder)
+                and module_info.module_finder.path == self._stdlib_path)
 
     def is_suggestion_match(self, module_name: str, prefix: str) -> bool:
         if prefix:

Lib/test/test_pyrepl/test_pyrepl.py

@@ -1,3 +1,4 @@
+import importlib
 import io
 import itertools
 import os
@@ -26,9 +27,16 @@
     code_to_events,
 )
 from _pyrepl.console import Event
-from _pyrepl._module_completer import ImportParser, ModuleCompleter
-from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig,
-                              _ReadlineWrapper)
+from _pyrepl._module_completer import (
+    ImportParser,
+    ModuleCompleter,
+    HARDCODED_SUBMODULES,
+)
+from _pyrepl.readline import (
+    ReadlineAlikeReader,
+    ReadlineConfig,
+    _ReadlineWrapper,
+)
 from _pyrepl.readline import multiline_input as readline_multiline_input
 
 try:
@@ -930,7 +938,6 @@ def test_func(self):
 
 class TestPyReplModuleCompleter(TestCase):
     def setUp(self):
-        import importlib
         # Make iter_modules() search only the standard library.
         # This makes the test more reliable in case there are
         # other user packages/scripts on PYTHONPATH which can
@@ -1013,14 +1020,6 @@ def test_sub_module_private_completions(self):
                 self.assertEqual(output, expected)
 
     def test_builtin_completion_top_level(self):
-        import importlib
-        # Make iter_modules() search only the standard library.
-        # This makes the test more reliable in case there are
-        # other user packages/scripts on PYTHONPATH which can
-        # intefere with the completions.
-        lib_path = os.path.dirname(importlib.__path__[0])
-        sys.path = [lib_path]
-
         cases = (
             ("import bui\t\n", "import builtins"),
             ("from bui\t\n", "from builtins"),
@@ -1076,6 +1075,32 @@ def test_no_fallback_on_regular_completion(self):
                 output = reader.readline()
                 self.assertEqual(output, expected)
 
+    def test_hardcoded_stdlib_submodules(self):
+        cases = (
+            ("import collections.\t\n", "import collections.abc"),
+            ("from os import \t\n", "from os import path"),
+            ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"),
+            ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"),
+        )
+        for code, expected in cases:
+            with self.subTest(code=code):
+                events = code_to_events(code)
+                reader = self.prepare_reader(events, namespace={})
+                output = reader.readline()
+                self.assertEqual(output, expected)
+
+    def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self):
+        with tempfile.TemporaryDirectory() as _dir:
+            dir = pathlib.Path(_dir)
+            (dir / "collections").mkdir()
+            (dir / "collections" / "__init__.py").touch()
+            (dir / "collections" / "foo.py").touch()
+            with patch.object(sys, "path", [dir, *sys.path]):
+                events = code_to_events("import collections.\t\n")
+                reader = self.prepare_reader(events, namespace={})
+                output = reader.readline()
+                self.assertEqual(output, "import collections.foo")
+
     def test_get_path_and_prefix(self):
         cases = (
             ('', ('', '')),
@@ -1204,6 +1229,19 @@ def test_parse_error(self):
             with self.subTest(code=code):
                 self.assertEqual(actual, None)
 
+
+class TestHardcodedSubmodules(TestCase):
+    def test_hardcoded_stdlib_submodules_are_importable(self):
+        for parent_path, submodules in HARDCODED_SUBMODULES.items():
+            for module_name in submodules:
+                path = f"{parent_path}.{module_name}"
+                with self.subTest(path=path):
+                    # We can't use importlib.util.find_spec here,
+                    # since some hardcoded submodules parents are
+                    # not proper packages
+                    importlib.import_module(path)
+
+
 class TestPasteEvent(TestCase):
     def prepare_reader(self, events):
         console = FakeConsole(events)

Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst

@@ -0,0 +1 @@
+Fix some standard library submodules missing from the :term:`REPL` auto-completion of imports.