python-docs-translations · m-aciek · Jan 13, 2025 · Jan 12, 2025 · Jan 12, 2025 · Jan 12, 2025
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -34,5 +34,11 @@ repos:
       - id: check-hooks-apply
       - id: check-useless-excludes
 
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.14.1
+    hooks:
+      - id: mypy
+        additional_dependencies: [types-docutils, types-polib, types-requests]
+
 ci:
   autoupdate_schedule: quarterly
diff --git a/generate.py b/generate.py
@@ -9,9 +9,11 @@
 # ]
 # ///
 import subprocess
+from collections.abc import Iterator
 from datetime import datetime, timezone
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import cast
 
 from git import Repo
 from jinja2 import Template
@@ -21,106 +23,48 @@
 import visitors
 from completion import branches_from_devguide, get_completion
 
-completion_progress = []
 generation_time = datetime.now(timezone.utc)
 
-with TemporaryDirectory() as clones_dir:
-    Repo.clone_from(
-        'https://github.com/python/devguide.git',
-        devguide_dir := Path(clones_dir, 'devguide'),
-        depth=1,
-    )
-    latest_branch = branches_from_devguide(devguide_dir)[0]
-    Repo.clone_from(
-        'https://github.com/python/cpython.git',
-        Path(clones_dir, 'cpython'),
-        depth=1,
-        branch=latest_branch,
-    )
-    subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True)
-    subprocess.run(
-        ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
-    )
-    languages_built = dict(build_status.get_languages())
-    for language, repo in repositories.get_languages_and_repos(devguide_dir):
-        if repo:
-            completion_number, translators_number = get_completion(clones_dir, repo)
-            visitors_number = visitors.get_number_of_visitors(language)
-        else:
-            completion_number, translators_number, visitors_number = 0.0, 0, 0
-        completion_progress.append(
-            (
-                language,
-                repo,
-                completion_number,
-                translators_number,
-                visitors_number,
-                language in languages_built,  # built on docs.python.org
-                languages_built.get(language),  # in switcher
-            )
+
+def get_completion_progress() -> (
+    Iterator[tuple[str, str, float, int, int, bool, bool | None]]
+):
+    with TemporaryDirectory() as clones_dir:
+        Repo.clone_from(
+            'https://github.com/python/devguide.git',
+            devguide_dir := Path(clones_dir, 'devguide'),
+            depth=1,
+        )
+        latest_branch = branches_from_devguide(devguide_dir)[0]
+        Repo.clone_from(
+            'https://github.com/python/cpython.git',
+            Path(clones_dir, 'cpython'),
+            depth=1,
+            branch=latest_branch,
         )
-        print(completion_progress[-1])
+        subprocess.run(
+            ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True
+        )
+        subprocess.run(
+            ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
+        )
+        languages_built = dict(build_status.get_languages())
+        for lang, repo in repositories.get_languages_and_repos(devguide_dir):
+            built = lang in languages_built
+            in_switcher = languages_built.get(lang)
+            if not repo:
+                yield lang, cast(str, repo), 0.0, 0, 0, built, in_switcher
+                continue
+            completion, translators = get_completion(clones_dir, repo)
+            visitors_num = visitors.get_number_of_visitors(lang) if built else 0
+            yield lang, repo, completion, translators, visitors_num, built, in_switcher
 
-template = Template(
-    """
-<html lang="en">
-<head>
-  <title>Python Docs Translation Dashboard</title>
-  <link rel="stylesheet" href="style.css">
-</head>
-<body>
-<h1>Python Docs Translation Dashboard</h1>
-<table>
-<thead>
-<tr>
-  <th>language</th>
-  <th>build</th>
-  <th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies">visitors<a/></th>
-  <th>translators</th>
-  <th>completion</th>
-</tr>
-</thead>
-<tbody>
-{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %}
-<tr>
-  {% if repo %}
-  <td data-label="language">
-    <a href="https://github.com/{{ repo }}" target="_blank">
-      {{ language }}
-    </a>
-  </td>
-  {% else %}
-  <td data-label="language">{{ language }}</td>
-  {% endif %}
-  <td data-label="build">
-    {% if build %}
-      <a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
-    {% else %}
-      ✗
-    {% endif %}
-  </td>
-  <td data-label="visitors">
-    <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
-      {{ '{:,}'.format(visitors) }}
-    </a>
-  </td>
-  <td data-label="translators">{{ '{:,}'.format(translators) }}</td>
-  <td data-label="completion">
-    <div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
-  </td>
-</tr>
-{% endfor %}
-</tbody>
-</table>
-<p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
-</body>
-</html>
-"""
-)
 
-output = template.render(
-    completion_progress=completion_progress, generation_time=generation_time
-)
+if __name__ == '__main__':
+    template = Template(Path('template.html.jinja').read_text())
+
+    output = template.render(
+        completion_progress=get_completion_progress(), generation_time=generation_time
+    )
 
-with open('index.html', 'w') as file:
-    file.write(output)
+    Path('index.html').write_text(output)
diff --git a/repositories.py b/repositories.py
@@ -1,23 +1,24 @@
-import pathlib
 import re
-from typing import Generator, Optional
+from collections.abc import Iterator
+from pathlib import Path
 
 from docutils import core
 from docutils.nodes import table, row
 
 
-def get_languages_and_repos(
-    devguide_dir: pathlib.Path,
-) -> Generator[tuple[str, Optional[str]], None, None]:
+def get_languages_and_repos(devguide_dir: Path) -> Iterator[tuple[str, str | None]]:
     translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
     doctree = core.publish_doctree(translating)
 
     for node in doctree.traverse(table):
         for row_node in node.traverse(row)[1:]:
             language = row_node[0].astext()
             repo = row_node[2].astext()
-            language_code = (
-                re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-')
-            )
+            language_match = re.match(r'.* \((.*)\)', language)
+            if not language_match:
+                raise ValueError(
+                    f'Expected a language code in brackets in devguide table, found {language}'
+                )
+            language_code = language_match.group(1).lower().replace('_', '-')
             repo_match = re.match(':github:`GitHub <(.*)>`', repo)
             yield language_code, repo_match and repo_match.group(1)
diff --git a/ruff.toml b/ruff.toml
@@ -1,2 +1,3 @@
 [format]
 quote-style = "single"
+skip-magic-trailing-comma = true
diff --git a/template.html.jinja b/template.html.jinja
@@ -0,0 +1,56 @@
+<html lang="en">
+<head>
+  <title>Python Docs Translation Dashboard</title>
+  <link rel="stylesheet" href="style.css">
+</head>
+<body>
+<h1>Python Docs Translation Dashboard</h1>
+<table>
+<thead>
+<tr>
+  <th>language</th>
+  <th>build</th>
+  <th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">visitors</a></th>
+  <th>translators</th>
+  <th>completion</th>
+</tr>
+</thead>
+<tbody>
+{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %}
+<tr>
+  {% if repo %}
+  <td data-label="language">
+    <a href="https://github.com/{{ repo }}" target="_blank">
+      {{ language }}
+    </a>
+  </td>
+  {% else %}
+  <td data-label="language">{{ language }}</td>
+  {% endif %}
+  <td data-label="build">
+    {% if build %}
+      <a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
+    {% else %}
+      ✗
+    {% endif %}
+  </td>
+  <td data-label="visitors">
+    {% if build %}
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
+        {{ '{:,}'.format(visitors) }}
+      </a>
+    {% else %}
+      0
+    {% endif %}
+  </td>
+  <td data-label="translators">{{ translators }}</td>
+  <td data-label="completion">
+    <div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
+  </td>
+</tr>
+{% endfor %}
+</tbody>
+</table>
+<p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
+</body>
+</html>
diff --git a/translators.py b/translators.py
@@ -1,4 +1,4 @@
-from collections.abc import Generator
+from collections.abc import Iterator
 from pathlib import Path
 
 from git import Repo
@@ -15,10 +15,10 @@ def get_number_from_git_history(path: Path) -> int:
     return len(Repo(path).git.shortlog('-s', 'HEAD').splitlines())
 
 
-def yield_from_headers(path: Path) -> Generator[str, None, None]:
+def yield_from_headers(path: Path) -> Iterator[str]:
     for file in path.rglob('*.po'):
         try:
-            header = pofile(file).header.splitlines()
+            header = pofile(file).header.splitlines()  # type: ignore[call-overload]  # https://github.com/python/typeshed/pull/13396
         except IOError:
             continue
         if 'Translators:' not in header:

diff --git a/visitors.py b/visitors.py
@@ -10,7 +10,7 @@ def get_number_of_visitors(language: str) -> int:
     param = urllib.parse.urlencode(
         {'filters': f'[["contains","event:page",["/{language}/"]]]'}
     )
-    r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=10)
+    r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=20)
     with (
         zipfile.ZipFile(io.BytesIO(r.content), 'r') as z,
         z.open('visitors.csv') as csv_file,