Simplify walk-modules.py (#181)

Author: AA-Turner

.github/workflows/listgen.yml

@@ -133,24 +133,11 @@ jobs:
       - name: walk modules
         env:
           LISTGEN_PYTHON_VERSION: ${{ matrix.python }}
-          LISTGEN_DRY_RUN: ${{ inputs.dry-run }}
-        run: |
-          python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt"
-
-          if [[ -f "./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt"" ]]; then
-            sort -u -o ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" \
-              ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" \
-              "${LISTGEN_PYTHON_VERSION}.txt"
-          else
-            sort -u -o ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt" \
-              "${LISTGEN_PYTHON_VERSION}.txt"
-          fi
-
-          rm "${LISTGEN_PYTHON_VERSION}.txt"
+        run: python ./support/walk-modules.py ./stdlib_list/lists/"${LISTGEN_PYTHON_VERSION}.txt"
 
-          if [[ "${LISTGEN_DRY_RUN}" == "true" ]]; then
-            git diff
-          fi
+      - name: show diff
+        if: ${{ inputs.dry-run }}
+        run: git diff
 
       - name: create PR
         if: ${{ !inputs.dry-run }}

support/walk-modules.py

@@ -10,26 +10,21 @@
 if TYPE_CHECKING:
     from collections.abc import Sequence
     from types import ModuleType
-    from typing import Protocol
-
-    class Writer(Protocol):
-        def write(self, s: str, /) -> object: ...
 
 SEEN_MODS = set()
 
 
-def walk_pkgutil(mod_name: str, locations: Sequence[str], io: Writer) -> None:
+def walk_pkgutil(mod_name: str, locations: Sequence[str]) -> None:
     for pkg in pkgutil.walk_packages(locations, f"{mod_name}."):
         if pkg.name in SEEN_MODS:
             continue
         else:
             # We don't recurse here because `walk_packages` takes care of
             # it for us.
             SEEN_MODS.add(pkg.name)
-            print(pkg.name, file=io)
 
 
-def walk_naive(mod_name: str, mod: ModuleType, io: Writer) -> None:
+def walk_naive(mod_name: str, mod: ModuleType) -> None:
     for attr in dir(mod):
         attr_obj = getattr(mod, attr, None)
         # Shouldn't happen, but who knows.
@@ -46,7 +41,7 @@ def walk_naive(mod_name: str, mod: ModuleType, io: Writer) -> None:
         try:
             submod_name = f"{mod_name}.{attr}"
             importlib.import_module(submod_name)
-            walk(submod_name, io)
+            walk(submod_name)
         except ImportError:
             # ...but sometimes we do want to include re-exports, since
             # they might be things like "accelerator" modules that don't
@@ -57,17 +52,16 @@ def walk_naive(mod_name: str, mod: ModuleType, io: Writer) -> None:
                 # that don't actually appear on disk. Experimentally,
                 # there are a few of these (like "TK").
                 importlib.import_module(attr)
-                walk(attr, io)
+                walk(attr)
             except ImportError:
                 continue
 
 
-def walk(mod_name: str, io: Writer) -> None:
+def walk(mod_name: str) -> None:
     if mod_name in SEEN_MODS:
         return
     else:
         SEEN_MODS.add(mod_name)
-        print(mod_name, file=io)
 
     # Try and import it.
     try:
@@ -81,26 +75,35 @@ def walk(mod_name: str, io: Writer) -> None:
         locations = None
 
     if locations is not None:
-        walk_pkgutil(mod_name, locations, io)
+        walk_pkgutil(mod_name, locations)
     else:
-        walk_naive(mod_name, mod, io)
+        walk_naive(mod_name, mod)
 
 
 if __name__ == "__main__":
     output = sys.argv[1]
 
-    with open(output, mode="w") as io:
-        for mod_name in sys.builtin_module_names:
-            walk(mod_name, io)
+    for mod_name in sys.builtin_module_names:
+        walk(mod_name)
 
-        if hasattr(sys, "stdlib_module_names"):
-            for mod_name in sys.stdlib_module_names:
-                walk(mod_name, io)
-        else:
-            for mod_name in sys.stdin:
-                # Our precomputed list might not start at the root, since it
-                # might be a package rather than a module.
-                if "." in mod_name:
-                    top_mod = mod_name.split(".")[0]
-                    walk(top_mod, io)
-                walk(mod_name.rstrip("\n"), io)
+    if hasattr(sys, "stdlib_module_names"):
+        for mod_name in sys.stdlib_module_names:
+            walk(mod_name)
+    else:
+        for mod_name in sys.stdin:
+            # Our precomputed list might not start at the root, since it
+            # might be a package rather than a module.
+            if "." in mod_name:
+                top_mod = mod_name.split(".")[0]
+                walk(top_mod)
+            walk(mod_name.rstrip("\n"))
+
+    try:
+        with open(output, encoding="utf-8") as io:
+            SEEN_MODS.update(io.read().splitlines())
+    except FileNotFoundError:
+        pass
+
+    with open(output, mode="w", encoding="utf-8") as io:
+        for line in sorted(SEEN_MODS):
+            print(line, file=io)