Metadata

.github/workflows/update.yml

@@ -1,6 +1,6 @@
 on:
   schedule:
-    - cron: '*/30 * * * *'
+    - cron: '17 * * * *'
   push:
     branches:
       - 'main'
@@ -19,7 +19,7 @@ jobs:
       - uses: astral-sh/setup-uv@v5
       - uses: actions/checkout@v4
       - run: sudo apt-get install -y gettext
-      - run: uv run generate.py  # generates "index.html"
+      - run: uv run generate.py  # generates index.html and index.json
       - run: mkdir -p build && cp index.* style.css build
       - name: Deploy 🚀
         if: github.event_name != 'pull_request'
@@ -53,9 +53,28 @@ jobs:
       - name: Debug index.html if pull request
         if: github.event_name == 'pull_request'
         run: |
-          curl -Lo index.html-public https://github.com/m-aciek/pydocs-translation-dashboard/raw/refs/heads/gh-pages/index.html
+          curl -Lo index.html-public https://github.com/python-docs-translations/dashboard/raw/refs/heads/gh-pages/index.html
           diff --color=always -u index.html-public index.html || :
           cat index.html
+      - run: uv run generate_metadata.py  # generates metadata.html
+      - run: cp metadata.html warnings* build
+      - name: Deploy metadata view 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build
+          clean: false
+          git-config-name: github-actions[bot]
+          git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
+      - name: Deploy metadata view to subdirectory if pull request 🚀
+        if: github.event_name == 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build
+          target-folder: ${{ github.ref_name }}
+          clean: false
+          git-config-name: github-actions[bot]
+          git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
       - uses: actions/upload-artifact@v4
         with:
           name: build

.gitignore

@@ -1 +1,4 @@
 index.html
+metadata.html
+warnings-*.txt
+clones

build_warnings.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from re import findall
+from shutil import copyfile
+
+import sphinx.cmd.build
+
+
+def number(clones_dir: str, repo: str, language_code: str) -> int:
+    language_part, *locale = language_code.split('-')
+    if locale:
+        lang_with_locale = f'{language_part}_{locale[0].upper()}'
+    else:
+        lang_with_locale = language_part
+    locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES')
+    locale_dir.mkdir(parents=True, exist_ok=True)
+    for po_file in (repo_dir := Path(clones_dir, 'translations', repo)).rglob('*.po'):
+        relative_path = po_file.relative_to(repo_dir)
+        target_file = locale_dir / relative_path
+        target_file.parent.mkdir(parents=True, exist_ok=True)
+        copyfile(po_file, target_file)
+    sphinx.cmd.build.main(
+        (
+            '--builder',
+            'html',
+            '--jobs',
+            'auto',
+            '--define',
+            f'language={language_code}',
+            '--verbose',
+            '--warning-file',
+            warning_file := f'{clones_dir}/warnings-{language_code}.txt',
+            f'{clones_dir}/cpython/Doc',  # sourcedir
+            f'./sphinxbuild/{language_code}',  # outputdir
+        )
+    )
+    copyfile(warning_file, f'warnings-{language_code}.txt')
+    return len(findall('ERROR|WARNING', Path(warning_file).read_text()))

completion.py

@@ -23,44 +23,52 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]:
 def get_completion(
     clones_dir: str, repo: str
 ) -> tuple[float, 'TranslatorsData', str, float]:
-    clone_path = Path(clones_dir, repo)
-    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
+    clone_path = Path(clones_dir, 'translations', repo)
+    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + [
+        'master',
+        'main',
+    ]:
         try:
             clone_repo = git.Repo.clone_from(
                 f'https://github.com/{repo}.git', clone_path, branch=branch
             )
         except git.GitCommandError:
             print(f'failed to clone {repo} {branch}')
             translators_data = TranslatorsData(0, False)
+            branch = ''
             continue
         else:
             translators_number = translators.get_number(clone_path)
             translators_link = translators.get_link(clone_path, repo, branch)
             translators_data = TranslatorsData(translators_number, translators_link)
             break
-    with TemporaryDirectory() as tmpdir:
-        completion = potodo.merge_and_scan_path(
-            clone_path,
-            pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
-            merge_path=Path(tmpdir),
-            hide_reserved=False,
-            api_url='',
-        ).completion
+    path_for_merge = Path(clones_dir, 'rebased_translations', repo)
+    completion = potodo.merge_and_scan_path(
+        clone_path,
+        pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
+        merge_path=path_for_merge,
+        hide_reserved=False,
+        api_url='',
+    ).completion
 
     if completion:
         # Fetch commit from before 30 days ago and checkout
-        commit = next(
-            clone_repo.iter_commits('HEAD', max_count=1, before='30 days ago')
-        )
-        clone_repo.git.checkout(commit.hexsha)
-        with TemporaryDirectory() as tmpdir:
-            month_ago_completion = potodo.merge_and_scan_path(
-                clone_path,
-                pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
-                merge_path=Path(tmpdir),
-                hide_reserved=False,
-                api_url='',
-            ).completion
+        try:
+            commit = next(
+                clone_repo.iter_commits('HEAD', max_count=1, before='30 days ago')
+            )
+        except StopIteration:
+            month_ago_completion = 0.0
+        else:
+            clone_repo.git.checkout(commit.hexsha)
+            with TemporaryDirectory() as tmpdir:
+                month_ago_completion = potodo.merge_and_scan_path(
+                    clone_path,
+                    pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
+                    merge_path=Path(tmpdir),
+                    hide_reserved=False,
+                    api_url='',
+                ).completion
     else:
         month_ago_completion = 0.0
 

generate.py

@@ -16,54 +16,51 @@
 from dataclasses import dataclass, asdict
 from datetime import datetime, timezone
 from pathlib import Path
-from tempfile import TemporaryDirectory
 
 from git import Repo
 from jinja2 import Template
 from urllib3 import PoolManager
 
-import contribute
 import build_status
+import contribute
 from completion import branches_from_devguide, get_completion, TranslatorsData
-from repositories import get_languages_and_repos, Language
+from repositories import Language, get_languages_and_repos
 
 generation_time = datetime.now(timezone.utc)
 
 
 def get_completion_progress() -> Iterator['LanguageProjectData']:
-    with TemporaryDirectory() as clones_dir:
-        Repo.clone_from(
-            'https://github.com/python/devguide.git',
-            devguide_dir := Path(clones_dir, 'devguide'),
-            depth=1,
-        )
-        latest_branch = branches_from_devguide(devguide_dir)[0]
-        Repo.clone_from(
-            'https://github.com/python/cpython.git',
-            cpython_dir := Path(clones_dir, 'cpython'),
-            depth=1,
-            branch=latest_branch,
-        )
-        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
-        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
-        languages_built = dict(build_status.get_languages(http := PoolManager()))
+    clones_dir = Path('clones')
+    Repo.clone_from(
+        'https://github.com/python/devguide.git',
+        devguide_dir := Path(clones_dir, 'devguide'),
+        depth=1,
+    )
+    latest_branch = branches_from_devguide(devguide_dir)[0]
+    Repo.clone_from(
+        'https://github.com/python/cpython.git',
+        cpython_dir := Path(clones_dir, 'cpython'),
+        depth=1,
+        branch=latest_branch,
+    )
+    subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
+    subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
+    languages_built = dict(build_status.get_languages(PoolManager()))
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            return executor.map(
-                get_project_data,
-                *zip(*get_languages_and_repos(devguide_dir)),
-                itertools.repeat(languages_built),
-                itertools.repeat(clones_dir),
-                itertools.repeat(http),
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        return executor.map(
+            get_project_data,
+            *zip(*get_languages_and_repos(devguide_dir)),
+            itertools.repeat(languages_built),
+            itertools.repeat(clones_dir),
+        )
 
 
 def get_project_data(
     language: Language,
     repo: str | None,
     languages_built: dict[str, bool],
     clones_dir: str,
-    http: PoolManager,
 ) -> 'LanguageProjectData':
     built = language.code in languages_built
     if repo:
@@ -72,7 +69,7 @@ def get_project_data(
         completion = 0.0
         translators_data = TranslatorsData(0, False)
         change = 0.0
-        branch = None
+        branch = ''
     return LanguageProjectData(
         language,
         repo,
@@ -91,7 +88,7 @@ def get_project_data(
 class LanguageProjectData:
     language: Language
     repository: str | None
-    branch: str | None
+    branch: str
     completion: float
     change: float
     translators: TranslatorsData

generate_metadata.py

@@ -0,0 +1,86 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+#     "gitpython",
+#     "potodo",
+#     "jinja2",
+#     "sphinx",
+#     "python-docs-theme",
+#     "dacite",
+#     "sphinx-lint",
+# ]
+# ///
+import concurrent.futures
+import itertools
+import logging
+from collections.abc import Iterator, Sequence
+from datetime import datetime, timezone
+from json import loads
+from pathlib import Path
+from sys import argv
+
+import dacite
+from git import Repo
+from jinja2 import Template
+from urllib3 import request
+
+import build_warnings
+import sphinx_lint
+from generate import LanguageProjectData
+from repositories import Language
+
+generation_time = datetime.now(timezone.utc)
+
+
+def get_projects_metadata(
+    completion_progress: Sequence[LanguageProjectData],
+) -> Iterator[tuple[int, int]]:
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        return executor.map(
+            get_metadata,
+            *zip(*map(get_language_repo_and_completion, completion_progress)),
+            itertools.repeat(Path('clones')),
+        )
+
+
+def get_metadata(
+    language: Language, repo: str | None, completion: float, clones_dir: str
+) -> tuple[int, int]:
+    if repo and (repo_path := Path(clones_dir, 'translations', repo)).exists():
+        Repo(repo_path).git.checkout()
+    return (
+        repo
+        and completion
+        and (
+            build_warnings.number(clones_dir, repo, language.code),
+            sphinx_lint.store_and_count_failures(clones_dir, repo, language.code),
+        )
+    ) or (0, 0)
+
+
+def get_language_repo_and_completion(
+    project: LanguageProjectData,
+) -> tuple[Language, str | None, float]:
+    return project.language, project.repository, project.completion
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    logging.info(f'starting at {generation_time}')
+    template = Template(Path('metadata.html.jinja').read_text())
+    if (index_path := Path('index.json')).exists():
+        index_json = loads(Path('index.json').read_text())
+    else:
+        index_json = request('GET', argv[1]).json()
+
+    completion_progress = [
+        dacite.from_dict(LanguageProjectData, project) for project in index_json
+    ]
+
+    output = template.render(
+        metadata=zip(completion_progress, get_projects_metadata(completion_progress)),
+        generation_time=generation_time,
+        duration=(datetime.now(timezone.utc) - generation_time).seconds,
+    )
+
+    Path('metadata.html').write_text(output)

metadata.html.jinja

@@ -0,0 +1,41 @@
+<html lang="en">
+<head>
+  <title>Python Docs Translation Dashboard</title>
+  <link rel="stylesheet" href="style.css">
+  <meta charset="UTF-8">
+  <base target="_blank">
+</head>
+<body>
+<h1>Python Docs Translation Dashboard</h1>
+<nav class="switchpages">
+<a href="index.html" target="_self">main</a> | meta
+</nav>
+<table>
+<thead>
+<tr>
+  <th>language</th>
+  <th>branch</th>
+  <th>build warnings*</th>
+  <th>lint failures</th>
+</tr>
+</thead>
+<tbody>
+{% for project, metadata in metadata | sort(attribute='0.completion,0.translators.number') | reverse %}
+<tr>
+  <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
+  <td data-label="branch">{{ project.branch }}</td>
+  <td data-label="warnings">
+    {% if project.completion %}<a href="warnings-{{ project.language.code }}.txt">{{ metadata[0] }}</a>{% else %}{{ metadata[0] }}{% endif %}
+  </td>
+  <td data-label="lint">
+    {% if project.completion %}<a href="warnings-lint-{{ project.language.code }}.txt">{{ metadata[1] }}</a>{% else %}{{ metadata[1] }}{% endif %}
+  </td>
+</tr>
+{% endfor %}
+</tbody>
+</table>
+<p>* number of Sphinx build process warnings</p>
+<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/">Python Developer’s Guide</a>.</p>
+<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
+</body>
+</html>