Get list of repositories from devguide

Author: m-aciek

completion.py

@@ -19,16 +19,15 @@ def branches_from_devguide() -> list[str]:
     ]
 
 
-def get_completion(clones_dir: str, language: str) -> float:
-    clone_path = Path(clones_dir, language)
-
-    for branch in branches_from_devguide():
+def get_completion(clones_dir: str, repo: str) -> float:
+    clone_path = Path(clones_dir, repo)
+    for branch in branches_from_devguide() + ['master']:
         try:
             git.Repo.clone_from(
-                f'https://github.com/python/python-docs-{language}.git', clone_path, depth=1, branch=branch
+                f'https://github.com/{repo}.git', clone_path, depth=1, branch=branch
             )
         except git.GitCommandError:
-            print(f'failed to clone {language} {branch}')
+            print(f'failed to clone {repo} {branch}')
             continue
         else:
             break

generate.py

@@ -4,6 +4,7 @@
 #     "potodo",
 #     "jinja2",
 #     "requests",
+#     "docutils",
 # ]
 # ///
 import subprocess
@@ -14,6 +15,7 @@
 from git import Repo
 from jinja2 import Template
 
+import repositories
 import visitors
 from completion import branches_from_devguide, get_completion
 
@@ -26,10 +28,13 @@
     )
     subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True)
     subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True)
-    for language in ('es', 'fr', 'id', 'it', 'ja', 'ko', 'pl', 'pt-br', 'tr', 'uk', 'zh-cn', 'zh-tw'):
-        completion_number = get_completion(clones_dir, language)
-        visitors_number = visitors.get_number_of_visitors(language)
-        completion_progress.append((language, completion_number, visitors_number))
+    for language, repo in repositories.get_languages_and_repos():
+        if repo:
+            completion_number = get_completion(clones_dir, repo)
+            visitors_number = visitors.get_number_of_visitors(language)
+        else:
+            completion_number, branch, visitors_number = 0., "", 0
+        completion_progress.append((language, repo, completion_number, visitors_number))
         print(completion_progress[-1])
 
 template = Template("""
@@ -49,10 +54,11 @@
 </tr>
 </thead>
 <tbody>
-{% for language, completion, visitors in completion_progress | sort(attribute=1) | reverse %}
+{% for language, repo, completion, visitors in completion_progress | sort(attribute=2) | reverse %}
 <tr>
+  {% if repo %}
   <td data-label="language">
-    <a href="https://github.com/python/python-docs-{{ language }}" target="_blank">
+    <a href="https://github.com/{{ repo }}" target="_blank">
       {{ language }}
     </a>
   </td>
@@ -61,6 +67,10 @@
       {{ '{:,}'.format(visitors) }}
     </a>
   </td>
+  {% else %}
+  <td data-label="language">{{ language }}</td>
+  <td data-label="visitors">0</td>
+  {% endif %}
   <td data-label="completion">
     <div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
   </td>

repositories.py

@@ -0,0 +1,22 @@
+import tempfile
+import pathlib
+import re
+from typing import Generator, Optional
+
+import git
+from docutils import core
+from docutils.nodes import table, row
+
+def get_languages_and_repos() -> Generator[tuple[str, Optional[str]], None, None]:
+    with tempfile.TemporaryDirectory() as clone_path:
+        git.Repo.clone_from(f'https://github.com/python/devguide.git', clone_path, depth=1)
+        translating = pathlib.Path(clone_path, 'documentation/translating.rst').read_text()
+    doctree = core.publish_doctree(translating)
+
+    for node in doctree.traverse(table):
+        for row_node in node.traverse(row)[1:]:
+            language = row_node[0].astext()
+            repo = row_node[2].astext()
+            language_code = re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-')
+            repo_match = re.match(':github:`GitHub <(.*)>`', repo)
+            yield language_code, repo_match and repo_match.group(1)