Improve auto-update of extern list

Author: abravalheri

pkg_resources/extern/__init__.py

@@ -72,8 +72,8 @@ def install(self):
 
 # [[[cog
 # import cog
-# from tools.vendored import yield_root_package
-# names = "\n".join(f"    {x!r}," for x in yield_root_package('pkg_resources'))
+# from tools.vendored import yield_top_level
+# names = "\n".join(f"    {x!r}," for x in yield_top_level('pkg_resources'))
 # cog.outl(f"names = (\n{names}\n)")
 # ]]]
 names = (

setuptools/extern/__init__.py

@@ -72,8 +72,8 @@ def install(self):
 
 # [[[cog
 # import cog
-# from tools.vendored import yield_root_package
-# names = "\n".join(f"    {x!r}," for x in yield_root_package('setuptools'))
+# from tools.vendored import yield_top_level
+# names = "\n".join(f"    {x!r}," for x in yield_top_level('setuptools'))
 # cog.outl(f"names = (\n{names}\n)")
 # ]]]
 names = (

tools/vendored.py

@@ -165,18 +165,23 @@ def update_setuptools():
     rewrite_more_itertools(vendor / "more_itertools")
 
 
-def yield_root_package(name):
-    """Useful when defining the MetaPathFinder
-    >>> examples = set(yield_root_package("setuptools")) & {"jaraco", "backports"}
+def yield_top_level(name):
+    """Iterate over all modules and (top level) packages vendored
+    >>> roots = set(yield_top_level("setuptools"))
+    >>> examples = roots & {"jaraco", "backports", "zipp"}
     >>> list(sorted(examples))
-    ['backports', 'jaraco']
-    """
-    vendored = Path(f"{name}/_vendor/vendored.txt")
-    yield from (
-        line.partition("=")[0].partition(".")[0].replace("-", "_")
-        for line in vendored.read_text(encoding="utf-8").splitlines()
-        if line and not line.startswith("#")
-    )
+    ['backports', 'jaraco', 'zipp']
+    """
+    vendor = Path(f"{name}/_vendor")
+    ignore = {"__pycache__", "__init__.py", ".ruff_cache"}
+
+    for item in vendor.iterdir():
+        if item.name in ignore:
+            continue
+        if item.is_dir() and item.suffix != ".dist-info":
+            yield str(item.name)
+        if item.is_file() and item.suffix == ".py":
+            yield str(item.stem)
 
 
 __name__ == '__main__' and update_vendored()