Get repos from devguide

Author: m-aciek

completion.py

@@ -18,19 +18,19 @@ def branches_from_devguide() -> list[str]:
     ]
 
 
-def get_completion_and_branch(tmpdir: str, language: str) -> tuple[float, str]:
-    clone_path = pathlib.Path(tmpdir, language)
-
-    for branch in branches_from_devguide():
+def get_completion_and_branch(tmpdir: str, repo: str) -> tuple[float, str]:
+    clone_path = pathlib.Path(tmpdir, repo)
+    completion = 0.
+    for branch in branches_from_devguide() + ["master"]:
         try:
-            git.Repo.clone_from(f'https://github.com/python/python-docs-{language}.git', clone_path, depth=1, branch=branch)
+            git.Repo.clone_from(f'https://github.com/{repo}.git', clone_path, depth=1, branch=branch)
         except git.GitCommandError:
-            print(f'failed to clone {language} {branch}')
+            print(f'failed to clone {repo} {branch}')
             continue
         try:
             completion = potodo.scan_path(clone_path, no_cache=True, hide_reserved=False, api_url='').completion
         except OSError:
-            print(f'failed to scan {language} {branch}')
+            print(f'failed to scan {repo} {branch}')
             shutil.rmtree(clone_path)
             continue
         else:

generate.py

@@ -4,6 +4,7 @@
 #     "potodo",
 #     "jinja2",
 #     "requests",
+#     "docutils",
 # ]
 # ///
 from datetime import datetime, timezone
@@ -12,16 +13,20 @@
 from jinja2 import Template
 
 import completion
+import repositories
 import visitors
 
 completion_progress = []
 generation_time = datetime.now(timezone.utc)
 
 with TemporaryDirectory() as tmpdir:
-    for language in ('es', 'fr', 'id', 'it', 'ja', 'ko', 'pl', 'pt-br', 'tr', 'uk', 'zh-cn', 'zh-tw'):
-        completion_number, branch = completion.get_completion_and_branch(tmpdir, language)
-        visitors_number = visitors.get_number_of_visitors(language)
-        completion_progress.append((language, completion_number, branch, visitors_number))
+    for language, repo in source.get_languages_and_repos():
+        if repo:
+            completion_number, branch = completion.get_completion_and_branch(tmpdir, repo)
+            visitors_number = visitors.get_number_of_visitors(language)
+        else:
+            completion_number, branch, visitors_number = 0., "", 0
+        completion_progress.append((language, repo, completion_number, branch, visitors_number))
         print(completion_progress[-1])
 
 template = Template("""
@@ -42,10 +47,11 @@
 </tr>
 </thead>
 <tbody>
-{% for language, completion, branch, visitors in completion_progress | sort(attribute=1) | reverse %}
+{% for language, repo, completion, branch, visitors in completion_progress | sort(attribute=2) | reverse %}
 <tr>
+  {% if repo %}
   <td data-label="language">
-    <a href="https://github.com/python/python-docs-{{ language }}" target="_blank">
+    <a href="https://github.com/{{ repo }}" target="_blank">
       {{ language }}
     </a>
   </td>
@@ -54,6 +60,10 @@
       {{ '{:,}'.format(visitors) }}
     </a>
   </td>
+  {% else %}
+  <td data-label="language">{{ language }}</td>
+  <td data-label="visitors">0</td>
+  {% endif %}
   <td data-label="branch">{{ branch }}</td>
   <td data-label="completion">
     <div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>

repositories.py

@@ -0,0 +1,22 @@
+import tempfile
+import pathlib
+import re
+from typing import Generator, Optional
+
+import git
+from docutils import core
+from docutils.nodes import table, row
+
+def get_languages_and_repos() -> Generator[tuple[str, Optional[str]], None, None]:
+    with tempfile.TemporaryDirectory() as clone_path:
+        git.Repo.clone_from(f'https://github.com/python/devguide.git', clone_path, depth=1)
+        translating = pathlib.Path(clone_path, 'documentation/translating.rst').read_text()
+    doctree = core.publish_doctree(translating)
+
+    for node in doctree.traverse(table):
+        for row_node in node.traverse(row)[1:]:
+            language = row_node[0].astext()
+            repo = row_node[2].astext()
+            language_code = re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-')
+            repo_match = re.match(':github:`GitHub <(.*)>`', repo)
+            yield language_code, repo_match and repo_match.group(1)