From 680812a901846fcbc120d832d8faaa56fba07f28 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 19 Aug 2025 17:00:00 +0100 Subject: [PATCH 1/2] Use ``submodule_search_locations`` correctly --- support/walk-modules.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/support/walk-modules.py b/support/walk-modules.py index 114e2ec..9b73cec 100755 --- a/support/walk-modules.py +++ b/support/walk-modules.py @@ -78,9 +78,12 @@ def walk(mod_name: str, io: Writer) -> None: try: locations = mod.__spec__.submodule_search_locations except AttributeError: - walk_naive(mod_name, mod, io) - else: + locations = None + + if locations is not None: walk_pkgutil(mod_name, locations, io) + else: + walk_naive(mod_name, mod, io) if __name__ == "__main__": From dc40d6df871e0a151acd7f6d28f129cb7b0b146d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 19 Aug 2025 17:22:55 +0100 Subject: [PATCH 2/2] Simplify ``walk-modules.py`` --- .github/workflows/listgen.yml | 21 +++---------- support/walk-modules.py | 57 ++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/.github/workflows/listgen.yml b/.github/workflows/listgen.yml index f14c6ee..f2d4399 100644 --- a/.github/workflows/listgen.yml +++ b/.github/workflows/listgen.yml @@ -133,24 +133,11 @@ jobs: - name: walk modules env: LISTGEN_PYTHON_VERSION: ${{ matrix.python }} - LISTGEN_DRY_RUN: ${{ inputs.dry-run }} - run: | - python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt" - - if [[ -f "./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt"" ]]; then - sort -u -o ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" \ - ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" \ - "${LISTGEN_PYTHON_VERSION}.txt" - else - sort -u -o ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" \ - "${LISTGEN_PYTHON_VERSION}.txt" - fi - - rm "${LISTGEN_PYTHON_VERSION}.txt" + run: python ./support/walk-modules.py ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" - if [[ "${LISTGEN_DRY_RUN}" == "true" ]]; then - git diff - fi + - name: show diff + if: ${{ inputs.dry-run }} + run: git diff - name: create PR if: ${{ !inputs.dry-run }} diff --git a/support/walk-modules.py b/support/walk-modules.py index 9b73cec..625982a 100755 --- a/support/walk-modules.py +++ b/support/walk-modules.py @@ -10,15 +10,11 @@ if TYPE_CHECKING: from collections.abc import Sequence from types import ModuleType - from typing import Protocol - - class Writer(Protocol): - def write(self, s: str, /) -> object: ... SEEN_MODS = set() -def walk_pkgutil(mod_name: str, locations: Sequence[str], io: Writer) -> None: +def walk_pkgutil(mod_name: str, locations: Sequence[str]) -> None: for pkg in pkgutil.walk_packages(locations, f"{mod_name}."): if pkg.name in SEEN_MODS: continue @@ -26,10 +22,9 @@ def walk_pkgutil(mod_name: str, locations: Sequence[str], io: Writer) -> None: # We don't recurse here because `walk_packages` takes care of # it for us. SEEN_MODS.add(pkg.name) - print(pkg.name, file=io) -def walk_naive(mod_name: str, mod: ModuleType, io: Writer) -> None: +def walk_naive(mod_name: str, mod: ModuleType) -> None: for attr in dir(mod): attr_obj = getattr(mod, attr, None) # Shouldn't happen, but who knows. @@ -46,7 +41,7 @@ def walk_naive(mod_name: str, mod: ModuleType, io: Writer) -> None: try: submod_name = f"{mod_name}.{attr}" importlib.import_module(submod_name) - walk(submod_name, io) + walk(submod_name) except ImportError: # ...but sometimes we do want to include re-exports, since # they might be things like "accelerator" modules that don't @@ -57,17 +52,16 @@ def walk_naive(mod_name: str, mod: ModuleType, io: Writer) -> None: # that don't actually appear on disk. Experimentally, # there are a few of these (like "TK"). importlib.import_module(attr) - walk(attr, io) + walk(attr) except ImportError: continue -def walk(mod_name: str, io: Writer) -> None: +def walk(mod_name: str) -> None: if mod_name in SEEN_MODS: return else: SEEN_MODS.add(mod_name) - print(mod_name, file=io) # Try and import it. try: @@ -81,26 +75,35 @@ def walk(mod_name: str, io: Writer) -> None: locations = None if locations is not None: - walk_pkgutil(mod_name, locations, io) + walk_pkgutil(mod_name, locations) else: - walk_naive(mod_name, mod, io) + walk_naive(mod_name, mod) if __name__ == "__main__": output = sys.argv[1] - with open(output, mode="w") as io: - for mod_name in sys.builtin_module_names: - walk(mod_name, io) + for mod_name in sys.builtin_module_names: + walk(mod_name) - if hasattr(sys, "stdlib_module_names"): - for mod_name in sys.stdlib_module_names: - walk(mod_name, io) - else: - for mod_name in sys.stdin: - # Our precomputed list might not start at the root, since it - # might be a package rather than a module. - if "." in mod_name: - top_mod = mod_name.split(".")[0] - walk(top_mod, io) - walk(mod_name.rstrip("\n"), io) + if hasattr(sys, "stdlib_module_names"): + for mod_name in sys.stdlib_module_names: + walk(mod_name) + else: + for mod_name in sys.stdin: + # Our precomputed list might not start at the root, since it + # might be a package rather than a module. + if "." in mod_name: + top_mod = mod_name.split(".")[0] + walk(top_mod) + walk(mod_name.rstrip("\n")) + + try: + with open(output, encoding="utf-8") as io: + SEEN_MODS.update(io.read().splitlines()) + except FileNotFoundError: + pass + + with open(output, mode="w", encoding="utf-8") as io: + for line in sorted(SEEN_MODS): + print(line, file=io)