Refactor, improvements

completion.py

@@ -1,8 +1,9 @@
 import json
+from collections.abc import Iterator
 from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Literal
+from typing import Literal, NamedTuple
 
 import git
 from potodo import potodo
@@ -11,31 +12,40 @@
 
 
 @cache
-def branches_from_devguide(devguide_dir: Path) -> list[str]:
+def latest_branch_from_devguide(devguide_dir: Path) -> str:
     p = devguide_dir.joinpath('include/release-cycle.json')
-    data = json.loads(p.read_text())
-    return [
-        branch for branch in data if data[branch]['status'] in ('bugfix', 'security')
-    ]
+    for branch in (data := json.loads(p.read_text())):
+        if data[branch]['status'] in ('bugfix', 'security'):
+            return branch
+    raise ValueError(f'Supported release not found in {p}')
 
 
-def get_completion(
-    clones_dir: str, repo: str
-) -> tuple[float, int, str | Literal[False]]:
+def iterate_branches(latest: str) -> Iterator[str]:
+    yield latest
+    major, minor = latest.split('.')
+    while int(minor) > 6:  # hu has 3.6 branch, hi has 3.7
+        minor = str(int(minor) - 1)
+        yield f'{major}.{minor}'
+    yield 'master'
+
+
+def get_completion(clones_dir: str, repo: str) -> tuple[float, 'TranslatorsData']:
     clone_path = Path(clones_dir, repo)
-    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
+    for branch in iterate_branches(
+        latest_branch_from_devguide(Path(clones_dir, 'devguide'))
+    ):
         try:
             git.Repo.clone_from(
                 f'https://github.com/{repo}.git', clone_path, branch=branch
             )
         except git.GitCommandError:
             print(f'failed to clone {repo} {branch}')
-            translators_number = 0
-            translators_link: str | Literal[False] = False
+            translators_data = TranslatorsData(0, False)
             continue
         else:
             translators_number = translators.get_number(clone_path)
             translators_link = translators.get_link(clone_path, repo, branch)
+            translators_data = TranslatorsData(translators_number, translators_link)
             break
     with TemporaryDirectory() as tmpdir:
         completion = potodo.merge_and_scan_path(
@@ -45,4 +55,14 @@ def get_completion(
             hide_reserved=False,
             api_url='',
         ).completion
-    return completion, translators_number, translators_link
+    return completion, translators_data
+
+
+class TranslatorsData(NamedTuple):
+    number: int
+    link: str | Literal[False]
+
+
+if __name__ == '__main__':
+    for branch in iterate_branches('3.13'):
+        print(branch)

generate.py

@@ -11,9 +11,10 @@
 import subprocess
 from collections.abc import Iterator
 from datetime import datetime, timezone
+from logging import info
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import cast, Literal
+from typing import cast, NamedTuple
 
 from git import Repo
 from jinja2 import Template
@@ -22,40 +23,24 @@
 import repositories
 import build_status
 import visitors
-from completion import branches_from_devguide, get_completion
+from completion import latest_branch_from_devguide, get_completion, TranslatorsData
+from repositories import Language
 
 generation_time = datetime.now(timezone.utc)
 
 
-def get_completion_progress() -> (
-    Iterator[
-        tuple[
-            str,
-            str,
-            str,
-            float,
-            int,
-            str | Literal[False],
-            int,
-            bool,
-            bool | None,
-            bool,
-            str | None,
-        ]
-    ]
-):
+def get_completion_progress() -> Iterator['LanguageProjectData']:
     with TemporaryDirectory() as clones_dir:
         Repo.clone_from(
             'https://github.com/python/devguide.git',
             devguide_dir := Path(clones_dir, 'devguide'),
             depth=1,
         )
-        latest_branch = branches_from_devguide(devguide_dir)[0]
         Repo.clone_from(
             'https://github.com/python/cpython.git',
             Path(clones_dir, 'cpython'),
             depth=1,
-            branch=latest_branch,
+            branch=latest_branch_from_devguide(devguide_dir),
         )
         subprocess.run(
             ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True
@@ -64,35 +49,33 @@ def get_completion_progress() -> (
             ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
         )
         languages_built = dict(build_status.get_languages())
-        for lang, lang_name, repo in repositories.get_languages_and_repos(devguide_dir):
-            built = lang in languages_built
-            in_switcher = languages_built.get(lang)
-            tx = lang in contribute.pulling_from_transifex
-            contrib_link = contribute.get_contrib_link(lang, repo)
+        for language, repo in repositories.get_languages_and_repos(devguide_dir):
+            built = language.code in languages_built
+            in_switcher = languages_built.get(language.code)
+            tx = language.code in contribute.pulling_from_transifex
+            contrib_link = contribute.get_contrib_link(language.code, repo)
             if not repo:
-                yield (
-                    lang,
-                    lang_name,
+                yield LanguageProjectData(
+                    language,
                     cast(str, repo),
                     0.0,
-                    0,
-                    False,
+                    TranslatorsData(0, False),
                     0,
                     built,
                     in_switcher,
                     False,
                     None,
                 )
                 continue
-            completion, translators, translators_link = get_completion(clones_dir, repo)
-            visitors_num = visitors.get_number_of_visitors(lang) if built else 0
-            yield (
-                lang,
-                lang_name,
+            completion, translators_data = get_completion(clones_dir, repo)
+            visitors_num = (
+                visitors.get_number_of_visitors(language.code) if built else 0
+            )
+            yield LanguageProjectData(
+                language,
                 repo,
                 completion,
-                translators,
-                translators_link,
+                translators_data,
                 visitors_num,
                 built,
                 in_switcher,
@@ -101,11 +84,26 @@ def get_completion_progress() -> (
             )
 
 
+class LanguageProjectData(NamedTuple):
+    language: Language
+    repository: str
+    completion: float
+    translators: 'TranslatorsData'
+    visitors: int
+    built: bool
+    in_switcher: bool | None
+    uses_platform: bool
+    contribution_link: str | None
+
+
 if __name__ == '__main__':
+    info(f'starting at {generation_time}')
     template = Template(Path('template.html.jinja').read_text())
 
     output = template.render(
-        completion_progress=get_completion_progress(), generation_time=generation_time
+        completion_progress=list(get_completion_progress()),
+        generation_time=generation_time,
+        duration=(datetime.now(timezone.utc) - generation_time).seconds,
     )
 
     Path('index.html').write_text(output)

repositories.py

@@ -2,6 +2,7 @@
 from collections.abc import Iterator
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import NamedTuple
 
 from docutils import core
 from docutils.nodes import table, row
@@ -10,7 +11,7 @@
 
 def get_languages_and_repos(
     devguide_dir: Path,
-) -> Iterator[tuple[str, str, str | None]]:
+) -> Iterator[tuple['Language', str | None]]:
     translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
     doctree = core.publish_doctree(translating)
 
@@ -26,7 +27,15 @@ def get_languages_and_repos(
             language_name = language_match.group(1)
             language_code = language_match.group(2).lower().replace('_', '-')
             repo_match = re.match(':github:`GitHub <(.*)>`', repo)
-            yield language_code, language_name, repo_match and repo_match.group(1)
+            yield (
+                Language(language_code, language_name),
+                repo_match and repo_match.group(1),
+            )
+
+
+class Language(NamedTuple):
+    code: str
+    name: str
 
 
 if __name__ == '__main__':

template.html.jinja

@@ -2,6 +2,7 @@
 <head>
   <title>Python Docs Translation Dashboard</title>
   <link rel="stylesheet" href="style.css">
+  <meta charset="UTF-8">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
@@ -17,44 +18,44 @@
 </tr>
 </thead>
 <tbody>
-{% for language, language_name, repo, completion, translators, translators_link, visitors, build, in_switcher, on_platform, contrib_link in completion_progress | sort(attribute='3,4') | reverse %}
+{% for project in completion_progress | sort(attribute='completion,translators.number') | reverse %}
 <tr>
-  <td data-label="language">{{ language_name }} ({{ language }})</td>
+  <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
   <td data-label="contribute">
-    {% if contrib_link %}<a href="{{ contrib_link }}" target="_blank">{% endif %}
-    {% if on_platform %}platform{% else %}repository{% endif %}
-    {% if contrib_link %}</a>{% endif %}
+    {% if project.contribution_link %}<a href="{{ project.contribution_link }}" target="_blank">{% endif %}
+    {% if project.uses_platform %}platform{% else %}repository{% endif %}
+    {% if project.contribution_link %}</a>{% endif %}
   </td>
   <td data-label="build">
-    {% if build %}
-      <a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
+    {% if project.built %}
+      <a href="https://docs.python.org/{{ project.language.code }}/" target="_blank">✓{% if project.in_switcher %} in switcher{% endif %}</a>
     {% else %}
       ✗
     {% endif %}
   </td>
   <td data-label="visitors">
-    {% if build %}
-      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
-        {{ '{:,}'.format(visitors) }}
+    {% if project.built %}
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))" target="_blank">
+        {{ '{:,}'.format(project.visitors) }}
       </a>
     {% else %}
-      {{ '{:,}'.format(visitors) }}
+      {{ '{:,}'.format(project.visitors) }}
     {% endif %}
   </td>
   <td data-label="translators">
-    {% if translators_link %}<a href="{{ translators_link }}" target="_blank">{% endif %}
-    {{ translators }}
-    {% if translators_link %}</a>{% endif %}
+    {% if project.translators.link %}<a href="{{ project.translators.link }}" target="_blank">{% endif %}
+    {{ project.translators.number }}
+    {% if project.translators.link %}</a>{% endif %}
   </td>
   <td data-label="completion">
-    <div class="progress-bar" style="width: {{ completion }}%;">{{ completion | round(2) }}%</div>
-    <div class="progress-bar-outer-label">{{ completion | round(2) }}%</div>
+    <div class="progress-bar" style="width: {{ project.completion }}%;">{{ "{:.2f}".format(project.completion) }}%</div>
+    <div class="progress-bar-outer-label">{{ "{:.2f}".format(project.completion) }}%</div>
   </td>
 </tr>
 {% endfor %}
 </tbody>
 </table>
-<p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
+<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
 </body>
 <script>
   function updateProgressBarVisibility() {

translators.py

@@ -1,5 +1,7 @@
 from collections.abc import Iterator
 from pathlib import Path
+from re import fullmatch
+from tempfile import TemporaryDirectory
 from typing import Literal
 
 from git import Repo
@@ -9,7 +11,8 @@
 def get_number(path: Path) -> int:
     from_headers = len(set(yield_from_headers(path)))
     from_git_history = get_number_from_git_history(path)
-    return max(from_headers, from_git_history)
+    from_translators_file = len(get_from_translators_file(path))
+    return max(from_headers, from_git_history, from_translators_file)
 
 
 def get_number_from_git_history(path: Path) -> int:
@@ -33,8 +36,44 @@ def yield_from_headers(path: Path) -> Iterator[str]:
                 yield translator
 
 
+def get_from_translators_file(path: Path) -> list[str]:
+    if not (file := path.joinpath('TRANSLATORS')).exists():
+        return []
+    return list(
+        line
+        for line in file.read_text().splitlines()
+        if line != 'Translators'
+        and not fullmatch(r'-*', line)
+        and not line.startswith('# ')
+    )
+
+
 def get_link(clone_path: Path, repo: str, branch: str) -> str | Literal[False]:
     return (
         clone_path.joinpath('TRANSLATORS').exists()
         and f'https://github.com/{repo}/blob/{branch}/TRANSLATORS'
     )
+
+
+if __name__ == '__main__':
+    for lang, branch in (
+        ('es', '3.13'),
+        ('hu', '3.6'),
+        ('pl', '3.13'),
+        ('zh-tw', '3.13'),
+        ('id', '3.9'),
+        ('fr', '3.13'),
+        ('hu', '3.6'),
+    ):
+        with TemporaryDirectory() as directory:
+            Repo.clone_from(
+                f'https://github.com/python/python-docs-{lang}',
+                directory,
+                branch=branch,
+            )
+            from_headers = len(set(yield_from_headers(path := Path(directory))))
+            from_git_history = get_number_from_git_history(path)
+            from_translators_file = len(get_from_translators_file(path))
+            print(
+                f'{lang}: {from_headers=}, {from_git_history=}, {from_translators_file=}'
+            )