Fix potential problem with cog script for vendored (#4353)

Author: abravalheri

pkg_resources/extern/__init__.py

@@ -72,18 +72,18 @@ def install(self):
 
 # [[[cog
 # import cog
-# from tools.vendored import yield_root_package
-# names = "\n".join(f"    {x!r}," for x in yield_root_package('pkg_resources'))
+# from tools.vendored import yield_top_level
+# names = "\n".join(f"    {x!r}," for x in yield_top_level('pkg_resources'))
 # cog.outl(f"names = (\n{names}\n)")
 # ]]]
 names = (
+    'backports',
+    'importlib_resources',
+    'jaraco',
+    'more_itertools',
     'packaging',
     'platformdirs',
-    'jaraco',
-    'importlib_resources',
     'zipp',
-    'more_itertools',
-    'backports',
 )
 # [[[end]]]
 VendorImporter(__name__, names).install()

setuptools/extern/__init__.py

@@ -72,20 +72,20 @@ def install(self):
 
 # [[[cog
 # import cog
-# from tools.vendored import yield_root_package
-# names = "\n".join(f"    {x!r}," for x in yield_root_package('setuptools'))
+# from tools.vendored import yield_top_level
+# names = "\n".join(f"    {x!r}," for x in yield_top_level('setuptools'))
 # cog.outl(f"names = (\n{names}\n)")
 # ]]]
 names = (
-    'packaging',
-    'ordered_set',
-    'more_itertools',
-    'jaraco',
-    'importlib_resources',
+    'backports',
     'importlib_metadata',
-    'zipp',
+    'importlib_resources',
+    'jaraco',
+    'more_itertools',
+    'ordered_set',
+    'packaging',
     'tomli',
-    'backports',
+    'zipp',
 )
 # [[[end]]]
 VendorImporter(__name__, names, 'setuptools._vendor').install()

tools/vendored.py

@@ -7,7 +7,7 @@
 
 def remove_all(paths):
     for path in paths:
-        path.rmtree() if path.isdir() else path.remove()
+        path.rmtree() if path.is_dir() else path.remove()
 
 
 def update_vendored():
@@ -165,18 +165,23 @@ def update_setuptools():
     rewrite_more_itertools(vendor / "more_itertools")
 
 
-def yield_root_package(name):
-    """Useful when defining the MetaPathFinder
-    >>> examples = set(yield_root_package("setuptools")) & {"jaraco", "backports"}
+def yield_top_level(name):
+    """Iterate over all modules and (top level) packages vendored
+    >>> roots = set(yield_top_level("setuptools"))
+    >>> examples = roots & {"jaraco", "backports", "zipp"}
     >>> list(sorted(examples))
-    ['backports', 'jaraco']
-    """
-    vendored = Path(f"{name}/_vendor/vendored.txt")
-    yield from (
-        line.partition("=")[0].partition(".")[0].replace("-", "_")
-        for line in vendored.read_text(encoding="utf-8").splitlines()
-        if line and not line.startswith("#")
-    )
+    ['backports', 'jaraco', 'zipp']
+    """
+    vendor = Path(f"{name}/_vendor")
+    ignore = {"__pycache__", "__init__.py", ".ruff_cache"}
+
+    for item in sorted(vendor.iterdir()):
+        if item.name in ignore:
+            continue
+        if item.is_dir() and item.suffix != ".dist-info":
+            yield str(item.name)
+        if item.is_file() and item.suffix == ".py":
+            yield str(item.stem)
 
 
 __name__ == '__main__' and update_vendored()

tox.ini

@@ -70,14 +70,14 @@ commands =
 
 [testenv:{vendor,check-extern}]
 skip_install = True
-allowlist_externals = sh
+allowlist_externals = git, sh
 deps =
 	path
 	cogapp
 commands =
 	vendor: python -m tools.vendored
-	vendor: sh -c "git grep -l -F '\[\[\[cog' | xargs cog -I {toxinidir} -r"  # update `*.extern`
-	check-extern: sh -c "git grep -l -F '\[\[\[cog' | xargs cog -I {toxinidir} --check"
+	sh -c "git grep -l -F '\[\[\[cog' | xargs -t cog -I {toxinidir} -r"  # update `*.extern`
+	check-extern: git diff --exit-code
 
 [testenv:generate-validation-code]
 skip_install = True