Get repos from devguide (python-docs-translations#18)

m-aciek · web-flow · commit 6e803e5c772c · 2024-12-14T21:04:37.000+01:00
* Get list of repositories from devguide

* Remove unused variable

* Fetch devguide once
diff --git a/completion.py b/completion.py
@@ -1,34 +1,29 @@
+import json
 from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
 import git
 from potodo import potodo
-import requests
 
 @cache
-def branches_from_devguide() -> list[str]:
-    r = requests.get(
-        "https://raw.githubusercontent.com/"
-        "python/devguide/main/include/release-cycle.json",
-        timeout=10,
-    )
-    data = r.json()
+def branches_from_devguide(devguide_dir: Path) -> list[str]:
+    p = devguide_dir.joinpath('include/release-cycle.json')
+    data = json.loads(p.read_text())
     return [
         branch for branch in data if data[branch]["status"] in ("bugfix", "security")
     ]
 
 
-def get_completion(clones_dir: str, language: str) -> float:
-    clone_path = Path(clones_dir, language)
-
-    for branch in branches_from_devguide():
+def get_completion(clones_dir: str, repo: str) -> float:
+    clone_path = Path(clones_dir, repo)
+    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
         try:
             git.Repo.clone_from(
-                f'https://github.com/python/python-docs-{language}.git', clone_path, depth=1, branch=branch
+                f'https://github.com/{repo}.git', clone_path, depth=1, branch=branch
             )
         except git.GitCommandError:
-            print(f'failed to clone {language} {branch}')
+            print(f'failed to clone {repo} {branch}')
             continue
         else:
             break
diff --git a/generate.py b/generate.py
@@ -4,6 +4,7 @@
 #     "potodo",
 #     "jinja2",
 #     "requests",
+#     "docutils",
 # ]
 # ///
 import subprocess
@@ -14,25 +15,32 @@
 from git import Repo
 from jinja2 import Template
 
+import repositories
 import visitors
 from completion import branches_from_devguide, get_completion
 
 completion_progress = []
 generation_time = datetime.now(timezone.utc)
 
 with TemporaryDirectory() as clones_dir:
+    Repo.clone_from(f'https://github.com/python/devguide.git', devguide_dir := Path(clones_dir, 'devguide'), depth=1)
+    latest_branch = branches_from_devguide(devguide_dir)[0]
     Repo.clone_from(
-        f'https://github.com/python/cpython.git', Path(clones_dir, 'cpython'), depth=1, branch=branches_from_devguide()[0]
+        f'https://github.com/python/cpython.git', Path(clones_dir, 'cpython'), depth=1, branch=latest_branch
     )
     subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True)
     subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True)
-    for language in ('es', 'fr', 'id', 'it', 'ja', 'ko', 'pl', 'pt-br', 'tr', 'uk', 'zh-cn', 'zh-tw'):
-        completion_number = get_completion(clones_dir, language)
-        visitors_number = visitors.get_number_of_visitors(language)
-        completion_progress.append((language, completion_number, visitors_number))
+    for language, repo in repositories.get_languages_and_repos(devguide_dir):
+        if repo:
+            completion_number = get_completion(clones_dir, repo)
+            visitors_number = visitors.get_number_of_visitors(language)
+        else:
+            completion_number, visitors_number = 0.0, 0
+        completion_progress.append((language, repo, completion_number, visitors_number))
         print(completion_progress[-1])
 
-template = Template("""
+template = Template(
+    """
 <html lang="en">
 <head>
   <title>Python Docs Translation Dashboard</title>
@@ -49,10 +57,11 @@
 </tr>
 </thead>
 <tbody>
-{% for language, completion, visitors in completion_progress | sort(attribute=1) | reverse %}
+{% for language, repo, completion, visitors in completion_progress | sort(attribute=2) | reverse %}
 <tr>
+  {% if repo %}
   <td data-label="language">
-    <a href="https://github.com/python/python-docs-{{ language }}" target="_blank">
+    <a href="https://github.com/{{ repo }}" target="_blank">
       {{ language }}
     </a>
   </td>
@@ -61,6 +70,10 @@
       {{ '{:,}'.format(visitors) }}
     </a>
   </td>
+  {% else %}
+  <td data-label="language">{{ language }}</td>
+  <td data-label="visitors">0</td>
+  {% endif %}
   <td data-label="completion">
     <div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
   </td>
@@ -71,7 +84,8 @@
 <p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
 </body>
 </html>
-""")
+"""
+)
 
 output = template.render(completion_progress=completion_progress, generation_time=generation_time)
 
diff --git a/repositories.py b/repositories.py
@@ -0,0 +1,20 @@
+import tempfile
+import pathlib
+import re
+from typing import Generator, Optional
+
+from docutils import core
+from docutils.nodes import table, row
+
+
+def get_languages_and_repos(devguide_dir: pathlib.Path) -> Generator[tuple[str, Optional[str]], None, None]:
+    translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
+    doctree = core.publish_doctree(translating)
+
+    for node in doctree.traverse(table):
+        for row_node in node.traverse(row)[1:]:
+            language = row_node[0].astext()
+            repo = row_node[2].astext()
+            language_code = re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-')
+            repo_match = re.match(':github:`GitHub <(.*)>`', repo)
+            yield language_code, repo_match and repo_match.group(1)
