Refactor, improvements

completion.py

@@ -1,4 +1,5 @@
 import json
+from dataclasses import dataclass
 from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -19,9 +20,7 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]:
     ]
 
 
-def get_completion(
-    clones_dir: str, repo: str
-) -> tuple[float, int, str | Literal[False]]:
+def get_completion(clones_dir: str, repo: str) -> tuple[float, 'TranslatorsData']:
     clone_path = Path(clones_dir, repo)
     for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
         try:
@@ -30,12 +29,12 @@ def get_completion(
             )
         except git.GitCommandError:
             print(f'failed to clone {repo} {branch}')
-            translators_number = 0
-            translators_link: str | Literal[False] = False
+            translators_data = TranslatorsData(0, False)
             continue
         else:
             translators_number = translators.get_number(clone_path)
             translators_link = translators.get_link(clone_path, repo, branch)
+            translators_data = TranslatorsData(translators_number, translators_link)
             break
     with TemporaryDirectory() as tmpdir:
         completion = potodo.merge_and_scan_path(
@@ -45,4 +44,10 @@ def get_completion(
             hide_reserved=False,
             api_url='',
         ).completion
-    return completion, translators_number, translators_link
+    return completion, translators_data
+
+
+@dataclass(frozen=True)
+class TranslatorsData:
+    number: int
+    link: str | Literal[False]

generate.py

@@ -10,40 +10,25 @@
 # ///
 import subprocess
 from collections.abc import Iterator
+from dataclasses import dataclass
 from datetime import datetime, timezone
+from logging import info
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import cast, Literal
 
 from git import Repo
 from jinja2 import Template
 
 import contribute
-import repositories
 import build_status
-import visitors
-from completion import branches_from_devguide, get_completion
+from visitors import get_number_of_visitors
+from completion import branches_from_devguide, get_completion, TranslatorsData
+from repositories import get_languages_and_repos, Language
 
 generation_time = datetime.now(timezone.utc)
 
 
-def get_completion_progress() -> (
-    Iterator[
-        tuple[
-            str,
-            str,
-            str,
-            float,
-            int,
-            str | Literal[False],
-            int,
-            bool,
-            bool | None,
-            bool,
-            str | None,
-        ]
-    ]
-):
+def get_completion_progress() -> Iterator['LanguageProjectData']:
     with TemporaryDirectory() as clones_dir:
         Repo.clone_from(
             'https://github.com/python/devguide.git',
@@ -53,59 +38,56 @@ def get_completion_progress() -> (
         latest_branch = branches_from_devguide(devguide_dir)[0]
         Repo.clone_from(
             'https://github.com/python/cpython.git',
-            Path(clones_dir, 'cpython'),
+            cpython_dir := Path(clones_dir, 'cpython'),
             depth=1,
             branch=latest_branch,
         )
-        subprocess.run(
-            ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True
-        )
-        subprocess.run(
-            ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
-        )
+        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
+        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
         languages_built = dict(build_status.get_languages())
-        for lang, lang_name, repo in repositories.get_languages_and_repos(devguide_dir):
-            built = lang in languages_built
-            in_switcher = languages_built.get(lang)
-            tx = lang in contribute.pulling_from_transifex
-            contrib_link = contribute.get_contrib_link(lang, repo)
-            if not repo:
-                yield (
-                    lang,
-                    lang_name,
-                    cast(str, repo),
-                    0.0,
-                    0,
-                    False,
-                    0,
-                    built,
-                    in_switcher,
-                    False,
-                    None,
-                )
-                continue
-            completion, translators, translators_link = get_completion(clones_dir, repo)
-            visitors_num = visitors.get_number_of_visitors(lang) if built else 0
-            yield (
-                lang,
-                lang_name,
+        for language, repo in get_languages_and_repos(devguide_dir):
+            built = language.code in languages_built
+            if repo:
+                completion, translators_data = get_completion(clones_dir, repo)
+                visitors_num = get_number_of_visitors(language.code) if built else 0
+            else:
+                completion = 0.0
+                translators_data = TranslatorsData(0, False)
+                visitors_num = 0
+            yield LanguageProjectData(
+                language,
                 repo,
                 completion,
-                translators,
-                translators_link,
+                translators_data,
                 visitors_num,
                 built,
-                in_switcher,
-                tx,
-                contrib_link,
+                in_switcher=languages_built.get(language.code),
+                uses_platform=language.code in contribute.pulling_from_transifex,
+                contribution_link=contribute.get_contrib_link(language.code, repo),
             )
 
 
+@dataclass(frozen=True)
+class LanguageProjectData:
+    language: Language
+    repository: str | None
+    completion: float
+    translators: TranslatorsData
+    visitors: int
+    built: bool
+    in_switcher: bool | None
+    uses_platform: bool
+    contribution_link: str | None
+
+
 if __name__ == '__main__':
+    info(f'starting at {generation_time}')
     template = Template(Path('template.html.jinja').read_text())
 
     output = template.render(
-        completion_progress=get_completion_progress(), generation_time=generation_time
+        completion_progress=list(get_completion_progress()),
+        generation_time=generation_time,
+        duration=(datetime.now(timezone.utc) - generation_time).seconds,
     )
 
     Path('index.html').write_text(output)

repositories.py

@@ -1,5 +1,6 @@
 import re
 from collections.abc import Iterator
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -10,7 +11,7 @@
 
 def get_languages_and_repos(
     devguide_dir: Path,
-) -> Iterator[tuple[str, str, str | None]]:
+) -> Iterator[tuple['Language', str | None]]:
     translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
     doctree = core.publish_doctree(translating)
 
@@ -26,7 +27,16 @@ def get_languages_and_repos(
             language_name = language_match.group(1)
             language_code = language_match.group(2).lower().replace('_', '-')
             repo_match = re.match(':github:`GitHub <(.*)>`', repo)
-            yield language_code, language_name, repo_match and repo_match.group(1)
+            yield (
+                Language(language_code, language_name),
+                repo_match and repo_match.group(1),
+            )
+
+
+@dataclass(frozen=True)
+class Language:
+    code: str
+    name: str
 
 
 if __name__ == '__main__':

template.html.jinja

@@ -2,6 +2,7 @@
 <head>
   <title>Python Docs Translation Dashboard</title>
   <link rel="stylesheet" href="style.css">
+  <meta charset="UTF-8">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
@@ -11,50 +12,52 @@
   <th>language</th>
   <th>contribute</th>
   <th>build</th>
-  <th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">visitors</a></th>
+  <th>visitors*</th>
   <th>translators</th>
   <th>completion</th>
 </tr>
 </thead>
 <tbody>
-{% for language, language_name, repo, completion, translators, translators_link, visitors, build, in_switcher, on_platform, contrib_link in completion_progress | sort(attribute='3,4') | reverse %}
+{% for project in completion_progress | sort(attribute='completion,translators.number') | reverse %}
 <tr>
-  <td data-label="language">{{ language_name }} ({{ language }})</td>
+  <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
   <td data-label="contribute">
-    {% if contrib_link %}<a href="{{ contrib_link }}" target="_blank">{% endif %}
-    {% if on_platform %}platform{% else %}repository{% endif %}
-    {% if contrib_link %}</a>{% endif %}
+    {% if project.contribution_link %}<a href="{{ project.contribution_link }}" target="_blank">{% endif %}
+    {% if project.uses_platform %}platform{% else %}repository{% endif %}
+    {% if project.contribution_link %}</a>{% endif %}
   </td>
   <td data-label="build">
-    {% if build %}
-      <a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
+    {% if project.built %}
+      <a href="https://docs.python.org/{{ project.language.code }}/" target="_blank">✓{% if project.in_switcher %} in switcher{% endif %}</a>
     {% else %}
       ✗
     {% endif %}
   </td>
   <td data-label="visitors">
-    {% if build %}
-      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
-        {{ '{:,}'.format(visitors) }}
+    {% if project.built %}
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))" target="_blank">
+        {{ '{:,}'.format(project.visitors) }}
       </a>
     {% else %}
-      {{ '{:,}'.format(visitors) }}
+      {{ '{:,}'.format(project.visitors) }}
     {% endif %}
   </td>
   <td data-label="translators">
-    {% if translators_link %}<a href="{{ translators_link }}" target="_blank">{% endif %}
-    {{ translators }}
-    {% if translators_link %}</a>{% endif %}
+    {% if project.translators.link %}<a href="{{ project.translators.link }}" target="_blank">{% endif %}
+    {{ project.translators.number }}
+    {% if project.translators.link %}</a>{% endif %}
   </td>
   <td data-label="completion">
-    <div class="progress-bar" style="width: {{ completion }}%;">{{ completion | round(2) }}%</div>
-    <div class="progress-bar-outer-label">{{ completion | round(2) }}%</div>
+    <div class="progress-bar" style="width: {{ project.completion }}%;">{{ "{:.2f}".format(project.completion) }}%</div>
+    <div class="progress-bar-outer-label">{{ "{:.2f}".format(project.completion) }}%</div>
   </td>
 </tr>
 {% endfor %}
 </tbody>
 </table>
-<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/" target="_blank">Python Developer’s Guide</a>. <br> Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
+<p>* sum of <a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">daily unique visitors</a> since 8 June 2024</p>
+<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/" target="_blank">Python Developer’s Guide</a>.</p>
+<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
 </body>
 <script>
   function updateProgressBarVisibility() {

translators.py

@@ -1,5 +1,7 @@
 from collections.abc import Iterator
 from pathlib import Path
+from re import fullmatch
+from tempfile import TemporaryDirectory
 from typing import Literal
 
 from git import Repo
@@ -9,7 +11,8 @@
 def get_number(path: Path) -> int:
     from_headers = len(set(yield_from_headers(path)))
     from_git_history = get_number_from_git_history(path)
-    return max(from_headers, from_git_history)
+    from_translators_file = len(get_from_translators_file(path))
+    return max(from_headers, from_git_history, from_translators_file)
 
 
 def get_number_from_git_history(path: Path) -> int:
@@ -33,8 +36,44 @@ def yield_from_headers(path: Path) -> Iterator[str]:
                 yield translator
 
 
+def get_from_translators_file(path: Path) -> list[str]:
+    if not (file := path.joinpath('TRANSLATORS')).exists():
+        return []
+    return list(
+        line
+        for line in file.read_text().splitlines()
+        if line != 'Translators'
+        and not fullmatch(r'-*', line)
+        and not line.startswith('# ')
+    )
+
+
 def get_link(clone_path: Path, repo: str, branch: str) -> str | Literal[False]:
     return (
         clone_path.joinpath('TRANSLATORS').exists()
         and f'https://github.com/{repo}/blob/{branch}/TRANSLATORS'
     )
+
+
+if __name__ == '__main__':
+    for lang, branch in (
+        ('es', '3.13'),
+        ('hu', '3.6'),
+        ('pl', '3.13'),
+        ('zh-tw', '3.13'),
+        ('id', '3.9'),
+        ('fr', '3.13'),
+        ('hu', '3.6'),
+    ):
+        with TemporaryDirectory() as directory:
+            Repo.clone_from(
+                f'https://github.com/python/python-docs-{lang}',
+                directory,
+                branch=branch,
+            )
+            from_headers = len(set(yield_from_headers(path := Path(directory))))
+            from_git_history = get_number_from_git_history(path)
+            from_translators_file = len(get_from_translators_file(path))
+            print(
+                f'{lang}: {from_headers=}, {from_git_history=}, {from_translators_file=}'
+            )