Get rid of cloning in generate_metadata.py

Author: m-aciek

build_warnings.py

@@ -12,9 +12,9 @@ def number(clones_dir: str, repo: str, language_code: str) -> int:
     else:
         lang_with_locale = language_part
     locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES')
-    locale_dir.mkdir(parents=True)
-    for po_file in Path(clones_dir, repo).rglob('*.po'):
-        relative_path = po_file.relative_to(Path(clones_dir, repo))
+    locale_dir.mkdir(parents=True, exist_ok=True)
+    for po_file in (repo_dir := Path(clones_dir, 'translations', repo)).rglob('*.po'):
+        relative_path = po_file.relative_to(repo_dir)
         target_file = locale_dir / relative_path
         target_file.parent.mkdir(parents=True, exist_ok=True)
         copyfile(po_file, target_file)

generate_metadata.py

@@ -13,23 +13,18 @@
 import concurrent.futures
 import itertools
 import logging
-import subprocess
 from collections.abc import Iterator, Sequence
 from datetime import datetime, timezone
 from json import loads
 from pathlib import Path
 from sys import argv
-from tempfile import TemporaryDirectory
 
 import dacite
-import git
-from git import Repo
 from jinja2 import Template
 from urllib3 import request
 
 import build_warnings
 import sphinx_lint
-from completion import branches_from_devguide
 from generate import LanguageProjectData
 from repositories import Language
 
@@ -39,41 +34,17 @@
 def get_projects_metadata(
     completion_progress: Sequence[LanguageProjectData],
 ) -> Iterator[tuple[int, int]]:
-    with TemporaryDirectory() as clones_dir:
-        Repo.clone_from(
-            'https://github.com/python/devguide.git',
-            devguide_dir := Path(clones_dir, 'devguide'),
-            depth=1,
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        return executor.map(
+            get_metadata,
+            *zip(*map(get_language_repo_and_completion, completion_progress)),
+            itertools.repeat(Path('clones')),
         )
-        latest_branch = branches_from_devguide(devguide_dir)[0]
-        Repo.clone_from(
-            'https://github.com/python/cpython.git',
-            cpython_dir := Path(clones_dir, 'cpython'),
-            depth=1,
-            branch=latest_branch,
-        )
-        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
-        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
-        with concurrent.futures.ProcessPoolExecutor() as executor:
-            return executor.map(
-                get_metadata,
-                *zip(
-                    *map(get_language_repo_branch_and_completion, completion_progress)
-                ),
-                itertools.repeat(clones_dir),
-            )
 
 
 def get_metadata(
-    language: Language,
-    repo: str | None,
-    branch: str | None,
-    completion: float,
-    clones_dir: str,
+    language: Language, repo: str | None, completion: float, clones_dir: str
 ) -> tuple[int, int]:
-    if repo:
-        clone_path = Path(clones_dir, repo)
-        git.Repo.clone_from(f'https://github.com/{repo}.git', clone_path, branch=branch)
     return (
         repo
         and completion
@@ -84,10 +55,10 @@ def get_metadata(
     ) or (0, 0)
 
 
-def get_language_repo_branch_and_completion(
+def get_language_repo_and_completion(
     project: LanguageProjectData,
-) -> tuple[Language, str | None, str | None, float]:
-    return project.language, project.repository, project.branch, project.completion
+) -> tuple[Language, str | None, float]:
+    return project.language, project.repository, project.completion
 
 
 if __name__ == '__main__':

sphinx_lint.py

@@ -14,5 +14,5 @@ def store_and_count_failures(clones_dir: str, repo: str, language_code: str) ->
 
 def yield_failures(clones_dir: str, repo: str) -> Iterator[str]:
     enabled_checkers = [c for c in checkers.all_checkers.values() if c.enabled]
-    for path in Path(clones_dir, repo).rglob('*.po'):
+    for path in Path(clones_dir, 'translations', repo).rglob('*.po'):
         yield check_file(path.as_posix(), enabled_checkers)