Refactor, introduce named tuples, iterate down to 3.6 to include Hungarian language progress, track generation time, ensure two decimal places in completion number, remove leading zero in date from day and hour, include TRANSLATORS file in translators number calc, add charset meta tag

Author: m-aciek

completion.py

@@ -1,8 +1,9 @@
 import json
+from collections.abc import Iterator
 from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Literal
+from typing import Literal, NamedTuple
 
 import git
 from potodo import potodo
@@ -11,31 +12,40 @@
 
 
 @cache
-def branches_from_devguide(devguide_dir: Path) -> list[str]:
+def latest_branch_from_devguide(devguide_dir: Path) -> str:
     p = devguide_dir.joinpath('include/release-cycle.json')
-    data = json.loads(p.read_text())
-    return [
-        branch for branch in data if data[branch]['status'] in ('bugfix', 'security')
-    ]
+    for branch in (data := json.loads(p.read_text())):
+        if data[branch]['status'] in ('bugfix', 'security'):
+            return branch
+    raise ValueError(f'Supported release not found in {p}')
 
 
-def get_completion(
-    clones_dir: str, repo: str
-) -> tuple[float, int, str | Literal[False]]:
+def iterate_branches(latest: str) -> Iterator[str]:
+    yield latest
+    major, minor = latest.split('.')
+    while int(minor) > 6:  # hu has 3.6 branch, hi has 3.7
+        minor = str(int(minor) - 1)
+        yield f'{major}.{minor}'
+    yield 'master'
+
+
+def get_completion(clones_dir: str, repo: str) -> tuple[float, 'TranslatorsData']:
     clone_path = Path(clones_dir, repo)
-    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
+    for branch in iterate_branches(
+        latest_branch_from_devguide(Path(clones_dir, 'devguide'))
+    ):
         try:
             git.Repo.clone_from(
                 f'https://github.com/{repo}.git', clone_path, branch=branch
             )
         except git.GitCommandError:
             print(f'failed to clone {repo} {branch}')
-            translators_number = 0
-            translators_link: str | Literal[False] = False
+            translators_data = TranslatorsData(0, False)
             continue
         else:
             translators_number = translators.get_number(clone_path)
             translators_link = translators.get_link(clone_path, repo, branch)
+            translators_data = TranslatorsData(translators_number, translators_link)
             break
     with TemporaryDirectory() as tmpdir:
         completion = potodo.merge_and_scan_path(
@@ -45,4 +55,14 @@ def get_completion(
             hide_reserved=False,
             api_url='',
         ).completion
-    return completion, translators_number, translators_link
+    return completion, translators_data
+
+
+class TranslatorsData(NamedTuple):
+    number: int
+    link: str | Literal[False]
+
+
+if __name__ == '__main__':
+    for branch in iterate_branches('3.13'):
+        print(branch)

generate.py

@@ -11,9 +11,10 @@
 import subprocess
 from collections.abc import Iterator
 from datetime import datetime, timezone
+from logging import info
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import cast, Literal
+from typing import cast, NamedTuple
 
 from git import Repo
 from jinja2 import Template
@@ -22,40 +23,24 @@
 import repositories
 import build_status
 import visitors
-from completion import branches_from_devguide, get_completion
+from completion import latest_branch_from_devguide, get_completion, TranslatorsData
+from repositories import Language
 
 generation_time = datetime.now(timezone.utc)
 
 
-def get_completion_progress() -> (
-    Iterator[
-        tuple[
-            str,
-            str,
-            str,
-            float,
-            int,
-            str | Literal[False],
-            int,
-            bool,
-            bool | None,
-            bool,
-            str | None,
-        ]
-    ]
-):
+def get_completion_progress() -> Iterator['LanguageProjectData']:
     with TemporaryDirectory() as clones_dir:
         Repo.clone_from(
             'https://github.com/python/devguide.git',
             devguide_dir := Path(clones_dir, 'devguide'),
             depth=1,
         )
-        latest_branch = branches_from_devguide(devguide_dir)[0]
         Repo.clone_from(
             'https://github.com/python/cpython.git',
             Path(clones_dir, 'cpython'),
             depth=1,
-            branch=latest_branch,
+            branch=latest_branch_from_devguide(devguide_dir),
         )
         subprocess.run(
             ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True
@@ -64,35 +49,33 @@ def get_completion_progress() -> (
             ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
         )
         languages_built = dict(build_status.get_languages())
-        for lang, lang_name, repo in repositories.get_languages_and_repos(devguide_dir):
-            built = lang in languages_built
-            in_switcher = languages_built.get(lang)
-            tx = lang in contribute.pulling_from_transifex
-            contrib_link = contribute.get_contrib_link(lang, repo)
+        for language, repo in repositories.get_languages_and_repos(devguide_dir):
+            built = language.code in languages_built
+            in_switcher = languages_built.get(language.code)
+            tx = language.code in contribute.pulling_from_transifex
+            contrib_link = contribute.get_contrib_link(language.code, repo)
             if not repo:
-                yield (
-                    lang,
-                    lang_name,
+                yield LanguageProjectData(
+                    language,
                     cast(str, repo),
                     0.0,
-                    0,
-                    False,
+                    TranslatorsData(0, False),
                     0,
                     built,
                     in_switcher,
                     False,
                     None,
                 )
                 continue
-            completion, translators, translators_link = get_completion(clones_dir, repo)
-            visitors_num = visitors.get_number_of_visitors(lang) if built else 0
-            yield (
-                lang,
-                lang_name,
+            completion, translators_data = get_completion(clones_dir, repo)
+            visitors_num = (
+                visitors.get_number_of_visitors(language.code) if built else 0
+            )
+            yield LanguageProjectData(
+                language,
                 repo,
                 completion,
-                translators,
-                translators_link,
+                translators_data,
                 visitors_num,
                 built,
                 in_switcher,
@@ -101,11 +84,26 @@ def get_completion_progress() -> (
             )
 
 
+class LanguageProjectData(NamedTuple):
+    language: Language
+    repository: str
+    completion: float
+    translators: 'TranslatorsData'
+    visitors: int
+    built: bool
+    in_switcher: bool | None
+    uses_platform: bool
+    contribution_link: str | None
+
+
 if __name__ == '__main__':
+    info(f'starting at {generation_time}')
     template = Template(Path('template.html.jinja').read_text())
 
     output = template.render(
-        completion_progress=get_completion_progress(), generation_time=generation_time
+        completion_progress=list(get_completion_progress()),
+        generation_time=generation_time,
+        duration=(datetime.now(timezone.utc) - generation_time).seconds,
     )
 
     Path('index.html').write_text(output)

repositories.py

@@ -2,6 +2,7 @@
 from collections.abc import Iterator
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import NamedTuple
 
 from docutils import core
 from docutils.nodes import table, row
@@ -10,7 +11,7 @@
 
 def get_languages_and_repos(
     devguide_dir: Path,
-) -> Iterator[tuple[str, str, str | None]]:
+) -> Iterator[tuple['Language', str | None]]:
     translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
     doctree = core.publish_doctree(translating)
 
@@ -26,7 +27,15 @@ def get_languages_and_repos(
             language_name = language_match.group(1)
             language_code = language_match.group(2).lower().replace('_', '-')
             repo_match = re.match(':github:`GitHub <(.*)>`', repo)
-            yield language_code, language_name, repo_match and repo_match.group(1)
+            yield (
+                Language(language_code, language_name),
+                repo_match and repo_match.group(1),
+            )
+
+
+class Language(NamedTuple):
+    code: str
+    name: str
 
 
 if __name__ == '__main__':

template.html.jinja

@@ -2,6 +2,7 @@
 <head>
   <title>Python Docs Translation Dashboard</title>
   <link rel="stylesheet" href="style.css">
+  <meta charset="UTF-8">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
@@ -17,44 +18,44 @@
 </tr>
 </thead>
 <tbody>
-{% for language, language_name, repo, completion, translators, translators_link, visitors, build, in_switcher, on_platform, contrib_link in completion_progress | sort(attribute='3,4') | reverse %}
+{% for project in completion_progress | sort(attribute='completion,translators.number') | reverse %}
 <tr>
-  <td data-label="language">{{ language_name }} ({{ language }})</td>
+  <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
   <td data-label="contribute">
-    {% if contrib_link %}<a href="{{ contrib_link }}" target="_blank">{% endif %}
-    {% if on_platform %}platform{% else %}repository{% endif %}
-    {% if contrib_link %}</a>{% endif %}
+    {% if project.contribution_link %}<a href="{{ project.contribution_link }}" target="_blank">{% endif %}
+    {% if project.uses_platform %}platform{% else %}repository{% endif %}
+    {% if project.contribution_link %}</a>{% endif %}
   </td>
   <td data-label="build">
-    {% if build %}
-      <a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
+    {% if project.built %}
+      <a href="https://docs.python.org/{{ project.language.code }}/" target="_blank">✓{% if project.in_switcher %} in switcher{% endif %}</a>
     {% else %}
       ✗
     {% endif %}
   </td>
   <td data-label="visitors">
-    {% if build %}
-      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
-        {{ '{:,}'.format(visitors) }}
+    {% if project.built %}
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))" target="_blank">
+        {{ '{:,}'.format(project.visitors) }}
       </a>
     {% else %}
-      {{ '{:,}'.format(visitors) }}
+      {{ '{:,}'.format(project.visitors) }}
     {% endif %}
   </td>
   <td data-label="translators">
-    {% if translators_link %}<a href="{{ translators_link }}" target="_blank">{% endif %}
-    {{ translators }}
-    {% if translators_link %}</a>{% endif %}
+    {% if project.translators.link %}<a href="{{ project.translators.link }}" target="_blank">{% endif %}
+    {{ project.translators.number }}
+    {% if project.translators.link %}</a>{% endif %}
   </td>
   <td data-label="completion">
-    <div class="progress-bar" style="width: {{ completion }}%;">{{ completion | round(2) }}%</div>
-    <div class="progress-bar-outer-label">{{ completion | round(2) }}%</div>
+    <div class="progress-bar" style="width: {{ project.completion }}%;">{{ "{:.2f}".format(project.completion) }}%</div>
+    <div class="progress-bar-outer-label">{{ "{:.2f}".format(project.completion) }}%</div>
   </td>
 </tr>
 {% endfor %}
 </tbody>
 </table>
-<p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
+<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
 </body>
 <script>
   function updateProgressBarVisibility() {

translators.py

@@ -1,5 +1,7 @@
 from collections.abc import Iterator
 from pathlib import Path
+from re import fullmatch
+from tempfile import TemporaryDirectory
 from typing import Literal
 
 from git import Repo
@@ -9,7 +11,8 @@
 def get_number(path: Path) -> int:
     from_headers = len(set(yield_from_headers(path)))
     from_git_history = get_number_from_git_history(path)
-    return max(from_headers, from_git_history)
+    from_translators_file = len(get_from_translators_file(path))
+    return max(from_headers, from_git_history, from_translators_file)
 
 
 def get_number_from_git_history(path: Path) -> int:
@@ -33,8 +36,44 @@ def yield_from_headers(path: Path) -> Iterator[str]:
                 yield translator
 
 
+def get_from_translators_file(path: Path) -> list[str]:
+    if not (file := path.joinpath('TRANSLATORS')).exists():
+        return []
+    return list(
+        line
+        for line in file.read_text().splitlines()
+        if line != 'Translators'
+        and not fullmatch(r'-*', line)
+        and not line.startswith('# ')
+    )
+
+
 def get_link(clone_path: Path, repo: str, branch: str) -> str | Literal[False]:
     return (
         clone_path.joinpath('TRANSLATORS').exists()
         and f'https://github.com/{repo}/blob/{branch}/TRANSLATORS'
     )
+
+
+if __name__ == '__main__':
+    for lang, branch in (
+        ('es', '3.13'),
+        ('hu', '3.6'),
+        ('pl', '3.13'),
+        ('zh-tw', '3.13'),
+        ('id', '3.9'),
+        ('fr', '3.13'),
+        ('hu', '3.6'),
+    ):
+        with TemporaryDirectory() as directory:
+            Repo.clone_from(
+                f'https://github.com/python/python-docs-{lang}',
+                directory,
+                branch=branch,
+            )
+            from_headers = len(set(yield_from_headers(path := Path(directory))))
+            from_git_history = get_number_from_git_history(path)
+            from_translators_file = len(get_from_translators_file(path))
+            print(
+                f'{lang}: {from_headers=}, {from_git_history=}, {from_translators_file=}'
+            )