Fixes to stubtest's new check for missing stdlib modules (#15960)

- It's not easy to predict where stdlib modules are going to be located.
(It varies between platforms, and between venvs and conda envs; on some
platforms it's in a completely different directory to the Python
executable.)
- Some modules appear to raise `SystemExit` when stubtest tries to
import them in CI, leading stubtest to instantly exit without logging a
message to the terminal.
- Importing some `test.*` submodules leads to unraisable exceptions
being printed to the terminal at the end of the stubtest run, which is
somewhat annoying.

Author: AlexWaygood

mypy/stubtest.py

@@ -23,6 +23,7 @@
 import typing
 import typing_extensions
 import warnings
+from collections import defaultdict
 from contextlib import redirect_stderr, redirect_stdout
 from functools import singledispatch
 from pathlib import Path
@@ -1679,16 +1680,22 @@ def get_importable_stdlib_modules() -> set[str]:
         all_stdlib_modules = sys.stdlib_module_names
     else:
         all_stdlib_modules = set(sys.builtin_module_names)
-        python_exe_dir = Path(sys.executable).parent
+        modules_by_finder: defaultdict[importlib.machinery.FileFinder, set[str]] = defaultdict(set)
         for m in pkgutil.iter_modules():
-            finder = m.module_finder
-            if isinstance(finder, importlib.machinery.FileFinder):
-                finder_path = Path(finder.path)
-                if (
-                    python_exe_dir in finder_path.parents
-                    and "site-packages" not in finder_path.parts
-                ):
-                    all_stdlib_modules.add(m.name)
+            if isinstance(m.module_finder, importlib.machinery.FileFinder):
+                modules_by_finder[m.module_finder].add(m.name)
+        for finder, module_group in modules_by_finder.items():
+            if (
+                "site-packages" not in Path(finder.path).parents
+                # if "_queue" is present, it's most likely the module finder
+                # for stdlib extension modules;
+                # if "queue" is present, it's most likely the module finder
+                # for pure-Python stdlib modules.
+                # In either case, we'll want to add all the modules that the finder has to offer us.
+                # This is a bit hacky, but seems to work well in a cross-platform way.
+                and {"_queue", "queue"} & module_group
+            ):
+                all_stdlib_modules.update(module_group)
 
     importable_stdlib_modules: set[str] = set()
     for module_name in all_stdlib_modules:
@@ -1719,13 +1726,25 @@ def get_importable_stdlib_modules() -> set[str]:
             # The idlelib.* submodules are similarly annoying in opening random tkinter windows,
             # and we're unlikely to ever add stubs for idlelib in typeshed
             # (see discussion in https://github.com/python/typeshed/pull/9193)
-            if submodule_name.endswith(".__main__") or submodule_name.startswith("idlelib."):
+            #
+            # test.* modules do weird things like raising exceptions in __del__ methods,
+            # leading to unraisable exceptions being logged to the terminal
+            # as a warning at the end of the stubtest run
+            if (
+                submodule_name.endswith(".__main__")
+                or submodule_name.startswith("idlelib.")
+                or submodule_name.startswith("test.")
+            ):
                 continue
 
             try:
                 silent_import_module(submodule_name)
+            except KeyboardInterrupt:
+                raise
             # importing multiprocessing.popen_forkserver on Windows raises AttributeError...
-            except Exception:
+            # some submodules also appear to raise SystemExit as well on some Python versions
+            # (not sure exactly which)
+            except BaseException:
                 continue
             else:
                 importable_stdlib_modules.add(submodule_name)