Refactor, improvements (#42)

Introduce dataclasses, track generation time, ensure two decimal places in completion number, remove leading zero in date from day and hour, include TRANSLATORS file in translators number calc, add charset meta tag, add visitors column comment, break bottom description into paragraphs, refactor code in main module

Author: m-aciek

completion.py

@@ -1,4 +1,5 @@
 import json
+from dataclasses import dataclass
 from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -19,9 +20,7 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]:
     ]
 
 
-def get_completion(
-    clones_dir: str, repo: str
-) -> tuple[float, int, str | Literal[False]]:
+def get_completion(clones_dir: str, repo: str) -> tuple[float, 'TranslatorsData']:
     clone_path = Path(clones_dir, repo)
     for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
         try:
@@ -30,12 +29,12 @@ def get_completion(
             )
         except git.GitCommandError:
             print(f'failed to clone {repo} {branch}')
-            translators_number = 0
-            translators_link: str | Literal[False] = False
+            translators_data = TranslatorsData(0, False)
             continue
         else:
             translators_number = translators.get_number(clone_path)
             translators_link = translators.get_link(clone_path, repo, branch)
+            translators_data = TranslatorsData(translators_number, translators_link)
             break
     with TemporaryDirectory() as tmpdir:
         completion = potodo.merge_and_scan_path(
@@ -45,4 +44,10 @@ def get_completion(
             hide_reserved=False,
             api_url='',
         ).completion
-    return completion, translators_number, translators_link
+    return completion, translators_data
+
+
+@dataclass(frozen=True)
+class TranslatorsData:
+    number: int
+    link: str | Literal[False]

generate.py

@@ -10,40 +10,25 @@
 # ///
 import subprocess
 from collections.abc import Iterator
+from dataclasses import dataclass
 from datetime import datetime, timezone
+from logging import info
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import cast, Literal
 
 from git import Repo
 from jinja2 import Template
 
 import contribute
-import repositories
 import build_status
-import visitors
-from completion import branches_from_devguide, get_completion
+from visitors import get_number_of_visitors
+from completion import branches_from_devguide, get_completion, TranslatorsData
+from repositories import get_languages_and_repos, Language
 
 generation_time = datetime.now(timezone.utc)
 
 
-def get_completion_progress() -> (
-    Iterator[
-        tuple[
-            str,
-            str,
-            str,
-            float,
-            int,
-            str | Literal[False],
-            int,
-            bool,
-            bool | None,
-            bool,
-            str | None,
-        ]
-    ]
-):
+def get_completion_progress() -> Iterator['LanguageProjectData']:
     with TemporaryDirectory() as clones_dir:
         Repo.clone_from(
             'https://github.com/python/devguide.git',
@@ -53,59 +38,56 @@ def get_completion_progress() -> (
         latest_branch = branches_from_devguide(devguide_dir)[0]
         Repo.clone_from(
             'https://github.com/python/cpython.git',
-            Path(clones_dir, 'cpython'),
+            cpython_dir := Path(clones_dir, 'cpython'),
             depth=1,
             branch=latest_branch,
         )
-        subprocess.run(
-            ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True
-        )
-        subprocess.run(
-            ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
-        )
+        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
+        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
         languages_built = dict(build_status.get_languages())
-        for lang, lang_name, repo in repositories.get_languages_and_repos(devguide_dir):
-            built = lang in languages_built
-            in_switcher = languages_built.get(lang)
-            tx = lang in contribute.pulling_from_transifex
-            contrib_link = contribute.get_contrib_link(lang, repo)
-            if not repo:
-                yield (
-                    lang,
-                    lang_name,
-                    cast(str, repo),
-                    0.0,
-                    0,
-                    False,
-                    0,
-                    built,
-                    in_switcher,
-                    False,
-                    None,
-                )
-                continue
-            completion, translators, translators_link = get_completion(clones_dir, repo)
-            visitors_num = visitors.get_number_of_visitors(lang) if built else 0
-            yield (
-                lang,
-                lang_name,
+        for language, repo in get_languages_and_repos(devguide_dir):
+            built = language.code in languages_built
+            if repo:
+                completion, translators_data = get_completion(clones_dir, repo)
+                visitors_num = get_number_of_visitors(language.code) if built else 0
+            else:
+                completion = 0.0
+                translators_data = TranslatorsData(0, False)
+                visitors_num = 0
+            yield LanguageProjectData(
+                language,
                 repo,
                 completion,
-                translators,
-                translators_link,
+                translators_data,
                 visitors_num,
                 built,
-                in_switcher,
-                tx,
-                contrib_link,
+                in_switcher=languages_built.get(language.code),
+                uses_platform=language.code in contribute.pulling_from_transifex,
+                contribution_link=contribute.get_contrib_link(language.code, repo),
             )
 
 
+@dataclass(frozen=True)
+class LanguageProjectData:
+    language: Language
+    repository: str | None
+    completion: float
+    translators: TranslatorsData
+    visitors: int
+    built: bool
+    in_switcher: bool | None
+    uses_platform: bool
+    contribution_link: str | None
+
+
 if __name__ == '__main__':
+    info(f'starting at {generation_time}')
     template = Template(Path('template.html.jinja').read_text())
 
     output = template.render(
-        completion_progress=get_completion_progress(), generation_time=generation_time
+        completion_progress=list(get_completion_progress()),
+        generation_time=generation_time,
+        duration=(datetime.now(timezone.utc) - generation_time).seconds,
     )
 
     Path('index.html').write_text(output)

repositories.py

@@ -1,5 +1,6 @@
 import re
 from collections.abc import Iterator
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -10,7 +11,7 @@
 
 def get_languages_and_repos(
     devguide_dir: Path,
-) -> Iterator[tuple[str, str, str | None]]:
+) -> Iterator[tuple['Language', str | None]]:
     translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
     doctree = core.publish_doctree(translating)
 
@@ -26,7 +27,16 @@ def get_languages_and_repos(
             language_name = language_match.group(1)
             language_code = language_match.group(2).lower().replace('_', '-')
             repo_match = re.match(':github:`GitHub <(.*)>`', repo)
-            yield language_code, language_name, repo_match and repo_match.group(1)
+            yield (
+                Language(language_code, language_name),
+                repo_match and repo_match.group(1),
+            )
+
+
+@dataclass(frozen=True)
+class Language:
+    code: str
+    name: str
 
 
 if __name__ == '__main__':

template.html.jinja

@@ -2,6 +2,7 @@
 <head>
   <title>Python Docs Translation Dashboard</title>
   <link rel="stylesheet" href="style.css">
+  <meta charset="UTF-8">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
@@ -11,50 +12,52 @@
   <th>language</th>
   <th>contribute</th>
   <th>build</th>
-  <th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">visitors</a></th>
+  <th>visitors*</th>
   <th>translators</th>
   <th>completion</th>
 </tr>
 </thead>
 <tbody>
-{% for language, language_name, repo, completion, translators, translators_link, visitors, build, in_switcher, on_platform, contrib_link in completion_progress | sort(attribute='3,4') | reverse %}
+{% for project in completion_progress | sort(attribute='completion,translators.number') | reverse %}
 <tr>
-  <td data-label="language">{{ language_name }} ({{ language }})</td>
+  <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
   <td data-label="contribute">
-    {% if contrib_link %}<a href="{{ contrib_link }}" target="_blank">{% endif %}
-    {% if on_platform %}platform{% else %}repository{% endif %}
-    {% if contrib_link %}</a>{% endif %}
+    {% if project.contribution_link %}<a href="{{ project.contribution_link }}" target="_blank">{% endif %}
+    {% if project.uses_platform %}platform{% else %}repository{% endif %}
+    {% if project.contribution_link %}</a>{% endif %}
   </td>
   <td data-label="build">
-    {% if build %}
-      <a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
+    {% if project.built %}
+      <a href="https://docs.python.org/{{ project.language.code }}/" target="_blank">✓{% if project.in_switcher %} in switcher{% endif %}</a>
     {% else %}
       ✗
     {% endif %}
   </td>
   <td data-label="visitors">
-    {% if build %}
-      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
-        {{ '{:,}'.format(visitors) }}
+    {% if project.built %}
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))" target="_blank">
+        {{ '{:,}'.format(project.visitors) }}
       </a>
     {% else %}
-      {{ '{:,}'.format(visitors) }}
+      {{ '{:,}'.format(project.visitors) }}
     {% endif %}
   </td>
   <td data-label="translators">
-    {% if translators_link %}<a href="{{ translators_link }}" target="_blank">{% endif %}
-    {{ translators }}
-    {% if translators_link %}</a>{% endif %}
+    {% if project.translators.link %}<a href="{{ project.translators.link }}" target="_blank">{% endif %}
+    {{ project.translators.number }}
+    {% if project.translators.link %}</a>{% endif %}
   </td>
   <td data-label="completion">
-    <div class="progress-bar" style="width: {{ completion }}%;">{{ completion | round(2) }}%</div>
-    <div class="progress-bar-outer-label">{{ completion | round(2) }}%</div>
+    <div class="progress-bar" style="width: {{ project.completion }}%;">{{ "{:.2f}".format(project.completion) }}%</div>
+    <div class="progress-bar-outer-label">{{ "{:.2f}".format(project.completion) }}%</div>
   </td>
 </tr>
 {% endfor %}
 </tbody>
 </table>
-<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/" target="_blank">Python Developer’s Guide</a>. <br> Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
+<p>* sum of <a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">daily unique visitors</a> since 8 June 2024</p>
+<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/" target="_blank">Python Developer’s Guide</a>.</p>
+<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
 </body>
 <script>
   function updateProgressBarVisibility() {

translators.py

@@ -1,5 +1,7 @@
 from collections.abc import Iterator
 from pathlib import Path
+from re import fullmatch
+from tempfile import TemporaryDirectory
 from typing import Literal
 
 from git import Repo
@@ -9,7 +11,8 @@
 def get_number(path: Path) -> int:
     from_headers = len(set(yield_from_headers(path)))
     from_git_history = get_number_from_git_history(path)
-    return max(from_headers, from_git_history)
+    from_translators_file = len(get_from_translators_file(path))
+    return max(from_headers, from_git_history, from_translators_file)
 
 
 def get_number_from_git_history(path: Path) -> int:
@@ -33,8 +36,44 @@ def yield_from_headers(path: Path) -> Iterator[str]:
                 yield translator
 
 
+def get_from_translators_file(path: Path) -> list[str]:
+    if not (file := path.joinpath('TRANSLATORS')).exists():
+        return []
+    return list(
+        line
+        for line in file.read_text().splitlines()
+        if line != 'Translators'
+        and not fullmatch(r'-*', line)
+        and not line.startswith('# ')
+    )
+
+
 def get_link(clone_path: Path, repo: str, branch: str) -> str | Literal[False]:
     return (
         clone_path.joinpath('TRANSLATORS').exists()
         and f'https://github.com/{repo}/blob/{branch}/TRANSLATORS'
     )
+
+
+if __name__ == '__main__':
+    for lang, branch in (
+        ('es', '3.13'),
+        ('hu', '3.6'),
+        ('pl', '3.13'),
+        ('zh-tw', '3.13'),
+        ('id', '3.9'),
+        ('fr', '3.13'),
+        ('hu', '3.6'),
+    ):
+        with TemporaryDirectory() as directory:
+            Repo.clone_from(
+                f'https://github.com/python/python-docs-{lang}',
+                directory,
+                branch=branch,
+            )
+            from_headers = len(set(yield_from_headers(path := Path(directory))))
+            from_git_history = get_number_from_git_history(path)
+            from_translators_file = len(get_from_translators_file(path))
+            print(
+                f'{lang}: {from_headers=}, {from_git_history=}, {from_translators_file=}'
+            )