Check all languages against latest source strings (python-docs-translations#6)

* Check all against local gettext build

* Remove branch info as superfluous

* Clone CPython repo and build gettext

* Add Sphinx dependency

* Add blurb dependency

* Fix clone path

* Debug log

* Install gettext in the workflow

* Format with black

* Skip OSError handling -- covered by potodo now

* Improve styling

* Simplify scan after upstream change

* Remove unused import

* Fix: break the versions loop when cloned

* Make cpython docs venv

* fix: bring back outer temp directory

* revert format on template

* update call argument name

* Remove custom potodo version and a note from template

* Install GNU Gettext in Read the Docs, reorder arguments

* Remove sudo call in Read the Docs command

* Add gettext with apt_packages (test)

* convert command to jobs

* move plugin add to install group

* Align visitors number to right

* Revert "Align visitors number to right"

This reverts commit 77be670.

* Add missing import

* Update completion.py: fix reference

* Add missing CPython clone code

Author: m-aciek

.github/workflows/update.yml

@@ -17,6 +17,7 @@ jobs:
           python-version: "3.x"
       - uses: astral-sh/setup-uv@v4
       - uses: actions/checkout@v4
+      - run: sudo apt-get install -y gettext
       - run: uv run generate.py  # generates "index.html"
       - run: mkdir -p build && cp index.html style.css build
       - name: Deploy 🚀

.readthedocs.yaml

@@ -8,10 +8,17 @@ build:
   os: ubuntu-24.04
   tools:
     python: "3"
-  commands:
-    - asdf plugin add uv
-    - asdf install uv latest
-    - asdf global uv latest
-    - uv run generate.py
-    - mkdir -p _readthedocs/html
-    - mv index.html style.css _readthedocs/html
+  apt_packages:
+    - gettext
+  jobs:
+    post_checkout: []
+    install:
+      - asdf plugin add uv
+      - asdf install uv latest
+    pre_build:
+      - asdf global uv latest
+    build:
+      html:
+        - uv run generate.py
+        - mkdir -p _readthedocs/html
+        - mv index.html style.css _readthedocs/html

completion.py

@@ -1,11 +1,12 @@
-import pathlib
-import shutil
+from functools import cache
+from pathlib import Path
+from tempfile import TemporaryDirectory
 
 import git
 from potodo import potodo
 import requests
 
-
+@cache
 def branches_from_devguide() -> list[str]:
     r = requests.get(
         "https://raw.githubusercontent.com/"
@@ -18,21 +19,21 @@ def branches_from_devguide() -> list[str]:
     ]
 
 
-def get_completion_and_branch(tmpdir: str, language: str) -> tuple[float, str]:
-    clone_path = pathlib.Path(tmpdir, language)
+def get_completion(clones_dir: str, language: str) -> float:
+    clone_path = Path(clones_dir, language)
 
     for branch in branches_from_devguide():
         try:
-            git.Repo.clone_from(f'https://github.com/python/python-docs-{language}.git', clone_path, depth=1, branch=branch)
+            git.Repo.clone_from(
+                f'https://github.com/python/python-docs-{language}.git', clone_path, depth=1, branch=branch
+            )
         except git.GitCommandError:
             print(f'failed to clone {language} {branch}')
             continue
-        try:
-            completion = potodo.scan_path(clone_path, no_cache=True, hide_reserved=False, api_url='').completion
-        except OSError:
-            print(f'failed to scan {language} {branch}')
-            shutil.rmtree(clone_path)
-            continue
         else:
             break
-    return completion, branch
+    with TemporaryDirectory() as tmpdir:
+        completion = potodo.merge_and_scan_path(
+            clone_path, pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'), merge_path=Path(tmpdir), hide_reserved=False, api_url=''
+        ).completion
+    return completion

generate.py

@@ -6,29 +6,37 @@
 #     "requests",
 # ]
 # ///
+import subprocess
 from datetime import datetime, timezone
+from pathlib import Path
 from tempfile import TemporaryDirectory
 
+from git import Repo
 from jinja2 import Template
 
-import completion
 import visitors
+from completion import branches_from_devguide, get_completion
 
 completion_progress = []
 generation_time = datetime.now(timezone.utc)
 
-with TemporaryDirectory() as tmpdir:
+with TemporaryDirectory() as clones_dir:
+    Repo.clone_from(
+        f'https://github.com/python/cpython.git', Path(clones_dir, 'cpython'), depth=1, branch=branches_from_devguide()[0]
+    )
+    subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True)
+    subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True)
     for language in ('es', 'fr', 'id', 'it', 'ja', 'ko', 'pl', 'pt-br', 'tr', 'uk', 'zh-cn', 'zh-tw'):
-        completion_number, branch = completion.get_completion_and_branch(tmpdir, language)
+        completion_number = get_completion(clones_dir, language)
         visitors_number = visitors.get_number_of_visitors(language)
-        completion_progress.append((language, completion_number, branch, visitors_number))
+        completion_progress.append((language, completion_number, visitors_number))
         print(completion_progress[-1])
 
 template = Template("""
 <html lang="en">
 <head>
-<title>Python Docs Translation Dashboard</title>
-<link rel="stylesheet" href="style.css">
+  <title>Python Docs Translation Dashboard</title>
+  <link rel="stylesheet" href="style.css">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
@@ -37,12 +45,11 @@
 <tr>
   <th>language</th>
   <th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies">visitors<a/></th>
-  <th>branch</th>
   <th>completion</th>
 </tr>
 </thead>
 <tbody>
-{% for language, completion, branch, visitors in completion_progress | sort(attribute=1) | reverse %}
+{% for language, completion, visitors in completion_progress | sort(attribute=1) | reverse %}
 <tr>
   <td data-label="language">
     <a href="https://github.com/python/python-docs-{{ language }}" target="_blank">
@@ -54,7 +61,6 @@
       {{ '{:,}'.format(visitors) }}
     </a>
   </td>
-  <td data-label="branch">{{ branch }}</td>
   <td data-label="completion">
     <div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
   </td>
@@ -63,7 +69,6 @@
 </tbody>
 </table>
 <p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
-<p>Note that the completion value is based on files available in language Git repository and <a href="https://github.com/m-aciek/pydocs-translation-dashboard/issues/2" target="_blank">may not include</a> e.g. resources which translation hasn't yet started.</p>
 </body>
 </html>
 """)