PyREPL module completion: hardcode special stdlib submodules

Author: loic-simon

Lib/_pyrepl/_module_completer.py

@@ -16,6 +16,15 @@
     from typing import Any, Iterable, Iterator, Mapping
 
 
+HARDCODED_SUBMODULES = {
+    # Standard library submodules that are not detected by pkgutil.iter_modules
+    # but can be imported, so should be proposed in completion
+    "collections": ["abc"],
+    "os": ["path"],
+    "xml.parsers.expat": ["errors", "model"],
+}
+
+
 def make_default_module_completer() -> ModuleCompleter:
     # Inside pyrepl, __package__ is set to None by default
     return ModuleCompleter(namespace={'__package__': None})
@@ -99,8 +108,14 @@ def _find_modules(self, path: str, prefix: str) -> list[str]:
             modules = [mod_info for mod_info in modules
                        if mod_info.ispkg and mod_info.name == segment]
             modules = self.iter_submodules(modules)
-        return [module.name for module in modules
-                if self.is_suggestion_match(module.name, prefix)]
+
+        module_names = [module.name for module in modules]
+        try:
+            module_names += HARDCODED_SUBMODULES[path]
+        except KeyError:
+            pass
+        return [module_name for module_name in module_names
+                if self.is_suggestion_match(module_name, prefix)]
 
     def is_suggestion_match(self, module_name: str, prefix: str) -> bool:
         if prefix:

Lib/test/test_pyrepl/test_pyrepl.py

@@ -1,3 +1,4 @@
+import importlib
 import io
 import itertools
 import os
@@ -26,7 +27,8 @@
     code_to_events,
 )
 from _pyrepl.console import Event
-from _pyrepl._module_completer import ImportParser, ModuleCompleter
+from _pyrepl._module_completer import (ImportParser, ModuleCompleter,
+                                       HARDCODED_SUBMODULES)
 from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig,
                               _ReadlineWrapper)
 from _pyrepl.readline import multiline_input as readline_multiline_input
@@ -930,7 +932,6 @@ def test_func(self):
 
 class TestPyReplModuleCompleter(TestCase):
     def setUp(self):
-        import importlib
         # Make iter_modules() search only the standard library.
         # This makes the test more reliable in case there are
         # other user packages/scripts on PYTHONPATH which can
@@ -1013,14 +1014,6 @@ def test_sub_module_private_completions(self):
                 self.assertEqual(output, expected)
 
     def test_builtin_completion_top_level(self):
-        import importlib
-        # Make iter_modules() search only the standard library.
-        # This makes the test more reliable in case there are
-        # other user packages/scripts on PYTHONPATH which can
-        # intefere with the completions.
-        lib_path = os.path.dirname(importlib.__path__[0])
-        sys.path = [lib_path]
-
         cases = (
             ("import bui\t\n", "import builtins"),
             ("from bui\t\n", "from builtins"),
@@ -1076,6 +1069,20 @@ def test_no_fallback_on_regular_completion(self):
                 output = reader.readline()
                 self.assertEqual(output, expected)
 
+    def test_hardcoded_stdlib_submodules(self):
+        cases = (
+            ("import collections.\t\n", "import collections.abc"),
+            ("from os import \t\n", "from os import path"),
+            ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"),
+            ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"),
+        )
+        for code, expected in cases:
+            with self.subTest(code=code):
+                events = code_to_events(code)
+                reader = self.prepare_reader(events, namespace={})
+                output = reader.readline()
+                self.assertEqual(output, expected)
+
     def test_get_path_and_prefix(self):
         cases = (
             ('', ('', '')),
@@ -1204,6 +1211,19 @@ def test_parse_error(self):
             with self.subTest(code=code):
                 self.assertEqual(actual, None)
 
+
+class TestHardcodedSubmodules(TestCase):
+    def test_hardcoded_stdlib_submodules_are_importable(self):
+        for parent_path, submodules in HARDCODED_SUBMODULES.items():
+            for module_name in submodules:
+                path = f"{parent_path}.{module_name}"
+                with self.subTest(path=path):
+                    # We can't use importlib.util.find_spec here,
+                    # since some hardcoded submodules parents are
+                    # not proper packages
+                    importlib.import_module(path)
+
+
 class TestPasteEvent(TestCase):
     def prepare_reader(self, events):
         console = FakeConsole(events)