diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml
index 84a0ea46a..377fafec8 100644
--- a/.github/workflows/update.yml
+++ b/.github/workflows/update.yml
@@ -1,6 +1,6 @@
on:
schedule:
- - cron: '*/30 * * * *'
+ - cron: '17 * * * *'
push:
branches:
- 'main'
@@ -19,7 +19,7 @@ jobs:
- uses: astral-sh/setup-uv@v5
- uses: actions/checkout@v4
- run: sudo apt-get install -y gettext
- - run: uv run generate.py # generates "index.html"
+ - run: uv run generate.py # generates index.html and index.json
- run: mkdir -p build && cp index.* style.css build
- name: Deploy 🚀
if: github.event_name != 'pull_request'
@@ -53,9 +53,28 @@ jobs:
- name: Debug index.html if pull request
if: github.event_name == 'pull_request'
run: |
- curl -Lo index.html-public https://github.com/m-aciek/pydocs-translation-dashboard/raw/refs/heads/gh-pages/index.html
+ curl -Lo index.html-public https://github.com/python-docs-translations/dashboard/raw/refs/heads/gh-pages/index.html
diff --color=always -u index.html-public index.html || :
cat index.html
+ - run: uv run generate_metadata.py # generates metadata.html
+ - run: cp metadata.html warnings* build
+ - name: Deploy metadata view 🚀
+ if: github.event_name != 'pull_request'
+ uses: JamesIves/github-pages-deploy-action@v4
+ with:
+ folder: build
+ clean: false
+ git-config-name: github-actions[bot]
+ git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
+ - name: Deploy metadata view to subdirectory if pull request 🚀
+ if: github.event_name == 'pull_request'
+ uses: JamesIves/github-pages-deploy-action@v4
+ with:
+ folder: build
+ target-folder: ${{ github.ref_name }}
+ clean: false
+ git-config-name: github-actions[bot]
+ git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
- uses: actions/upload-artifact@v4
with:
name: build
diff --git a/.gitignore b/.gitignore
index dcaf71693..c9a1725fd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,4 @@
index.html
+metadata.html
+warnings-*.txt
+clones
diff --git a/build_warnings.py b/build_warnings.py
new file mode 100644
index 000000000..23789b74a
--- /dev/null
+++ b/build_warnings.py
@@ -0,0 +1,37 @@
+from pathlib import Path
+from re import findall
+from shutil import copyfile
+
+import sphinx.cmd.build
+
+
+def number(clones_dir: str, repo: str, language_code: str) -> int:
+ language_part, *locale = language_code.split('-')
+ if locale:
+ lang_with_locale = f'{language_part}_{locale[0].upper()}'
+ else:
+ lang_with_locale = language_part
+ locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES')
+ locale_dir.mkdir(parents=True, exist_ok=True)
+ for po_file in (repo_dir := Path(clones_dir, 'translations', repo)).rglob('*.po'):
+ relative_path = po_file.relative_to(repo_dir)
+ target_file = locale_dir / relative_path
+ target_file.parent.mkdir(parents=True, exist_ok=True)
+ copyfile(po_file, target_file)
+ sphinx.cmd.build.main(
+ (
+ '--builder',
+ 'html',
+ '--jobs',
+ 'auto',
+ '--define',
+ f'language={language_code}',
+ '--verbose',
+ '--warning-file',
+ warning_file := f'{clones_dir}/warnings-{language_code}.txt',
+ f'{clones_dir}/cpython/Doc', # sourcedir
+ f'./sphinxbuild/{language_code}', # outputdir
+ )
+ )
+ copyfile(warning_file, f'warnings-{language_code}.txt')
+ return len(findall('ERROR|WARNING', Path(warning_file).read_text()))
diff --git a/completion.py b/completion.py
index cf1be877a..a0a0f840f 100644
--- a/completion.py
+++ b/completion.py
@@ -23,8 +23,11 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]:
def get_completion(
clones_dir: str, repo: str
) -> tuple[float, 'TranslatorsData', str, float]:
- clone_path = Path(clones_dir, repo)
- for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
+ clone_path = Path(clones_dir, 'translations', repo)
+ for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + [
+ 'master',
+ 'main',
+ ]:
try:
clone_repo = git.Repo.clone_from(
f'https://github.com/{repo}.git', clone_path, branch=branch
@@ -32,23 +35,24 @@ def get_completion(
except git.GitCommandError:
print(f'failed to clone {repo} {branch}')
translators_data = TranslatorsData(0, False)
+ branch = ''
continue
else:
translators_number = translators.get_number(clone_path)
translators_link = translators.get_link(clone_path, repo, branch)
translators_data = TranslatorsData(translators_number, translators_link)
break
- with TemporaryDirectory() as tmpdir:
- completion = potodo.merge_and_scan_path(
- clone_path,
- pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
- merge_path=Path(tmpdir),
- hide_reserved=False,
- api_url='',
- ).completion
+ path_for_merge = Path(clones_dir, 'rebased_translations', repo)
+ completion = potodo.merge_and_scan_path(
+ clone_path,
+ pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
+ merge_path=path_for_merge,
+ hide_reserved=False,
+ api_url='',
+ ).completion
if completion:
- # Fetch commit from before 30 days ago and checkout
+ # Get latest commit date and fetch commit from before 30 days ago and checkout
try:
commit = next(
clone_repo.iter_commits('HEAD', max_count=1, before='30 days ago')
diff --git a/generate.py b/generate.py
index d5fde7a4a..611914034 100644
--- a/generate.py
+++ b/generate.py
@@ -16,46 +16,44 @@
from dataclasses import dataclass, asdict
from datetime import datetime, timezone
from pathlib import Path
-from tempfile import TemporaryDirectory
from git import Repo
from jinja2 import Template
from urllib3 import PoolManager
-import contribute
import build_status
+import contribute
from completion import branches_from_devguide, get_completion, TranslatorsData
-from repositories import get_languages_and_repos, Language
+from repositories import Language, get_languages_and_repos
generation_time = datetime.now(timezone.utc)
def get_completion_progress() -> Iterator['LanguageProjectData']:
- with TemporaryDirectory() as clones_dir:
- Repo.clone_from(
- 'https://github.com/python/devguide.git',
- devguide_dir := Path(clones_dir, 'devguide'),
- depth=1,
- )
- latest_branch = branches_from_devguide(devguide_dir)[0]
- Repo.clone_from(
- 'https://github.com/python/cpython.git',
- cpython_dir := Path(clones_dir, 'cpython'),
- depth=1,
- branch=latest_branch,
- )
- subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
- subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
- languages_built = dict(build_status.get_languages(http := PoolManager()))
+ clones_dir = Path('clones')
+ Repo.clone_from(
+ 'https://github.com/python/devguide.git',
+ devguide_dir := Path(clones_dir, 'devguide'),
+ depth=1,
+ )
+ latest_branch = branches_from_devguide(devguide_dir)[0]
+ Repo.clone_from(
+ 'https://github.com/python/cpython.git',
+ cpython_dir := Path(clones_dir, 'cpython'),
+ depth=1,
+ branch=latest_branch,
+ )
+ subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
+ subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
+ languages_built = dict(build_status.get_languages(PoolManager()))
- with concurrent.futures.ThreadPoolExecutor() as executor:
- return executor.map(
- get_project_data,
- *zip(*get_languages_and_repos(devguide_dir)),
- itertools.repeat(languages_built),
- itertools.repeat(clones_dir),
- itertools.repeat(http),
- )
+ with concurrent.futures.ThreadPoolExecutor() as executor:
+ return executor.map(
+ get_project_data,
+ *zip(*get_languages_and_repos(devguide_dir)),
+ itertools.repeat(languages_built),
+ itertools.repeat(clones_dir),
+ )
def get_project_data(
@@ -63,7 +61,6 @@ def get_project_data(
repo: str | None,
languages_built: dict[str, bool],
clones_dir: str,
- http: PoolManager,
) -> 'LanguageProjectData':
built = language.code in languages_built
if repo:
@@ -72,7 +69,7 @@ def get_project_data(
completion = 0.0
translators_data = TranslatorsData(0, False)
change = 0.0
- branch = None
+ branch = ''
return LanguageProjectData(
language,
repo,
@@ -91,7 +88,7 @@ def get_project_data(
class LanguageProjectData:
language: Language
repository: str | None
- branch: str | None
+ branch: str
completion: float
change: float
translators: TranslatorsData
diff --git a/generate_metadata.py b/generate_metadata.py
new file mode 100644
index 000000000..30b72668f
--- /dev/null
+++ b/generate_metadata.py
@@ -0,0 +1,87 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+# "gitpython",
+# "potodo",
+# "jinja2",
+# "sphinx",
+# "python-docs-theme",
+# "dacite",
+# "sphinx-lint",
+# ]
+# ///
+import concurrent.futures
+import itertools
+import logging
+from collections.abc import Iterator, Sequence
+from datetime import datetime, timezone
+from json import loads
+from pathlib import Path
+from sys import argv
+
+import dacite
+from git import Repo
+from jinja2 import Template
+from urllib3 import request
+
+import build_warnings
+import sphinx_lint
+from generate import LanguageProjectData
+from repositories import Language
+
+generation_time = datetime.now(timezone.utc)
+
+
+def get_projects_metadata(
+ completion_progress: Sequence[LanguageProjectData],
+) -> Iterator[tuple[int, int, datetime | None]]:
+ with concurrent.futures.ProcessPoolExecutor() as executor:
+ return executor.map(
+ get_metadata,
+ *zip(*map(get_language_repo_and_completion, completion_progress)),
+ itertools.repeat(Path('clones')),
+ )
+
+
+def get_metadata(
+ language: Language, repo: str | None, completion: float, clones_dir: str
+) -> tuple[int, int, datetime | None]:
+ if not repo or not (repo_path := Path(clones_dir, 'translations', repo)).exists():
+ return 0, 0, None
+ (clone_repo := Repo(repo_path)).git.checkout()
+ latest_commit = clone_repo.head.commit.committed_datetime
+ if not completion:
+ return 0, 0, latest_commit
+ return (
+ build_warnings.number(clones_dir, repo, language.code),
+ sphinx_lint.store_and_count_failures(clones_dir, repo, language.code),
+ latest_commit,
+ )
+
+
+def get_language_repo_and_completion(
+ project: LanguageProjectData,
+) -> tuple[Language, str | None, float]:
+ return project.language, project.repository, project.completion
+
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.INFO)
+ logging.info(f'starting at {generation_time}')
+ template = Template(Path('metadata.html.jinja').read_text())
+ if (index_path := Path('index.json')).exists():
+ index_json = loads(Path('index.json').read_text())
+ else:
+ index_json = request('GET', argv[1]).json()
+
+ completion_progress = [
+ dacite.from_dict(LanguageProjectData, project) for project in index_json
+ ]
+
+ output = template.render(
+ metadata=zip(completion_progress, get_projects_metadata(completion_progress)),
+ generation_time=generation_time,
+ duration=(datetime.now(timezone.utc) - generation_time).seconds,
+ )
+
+ Path('metadata.html').write_text(output)
diff --git a/metadata.html.jinja b/metadata.html.jinja
new file mode 100644
index 000000000..d79f64dbe
--- /dev/null
+++ b/metadata.html.jinja
@@ -0,0 +1,43 @@
+
+
+ Python Docs Translation Dashboard
+
+
+
+
+
+Python Docs Translation Dashboard
+
+
+
+
+ | language |
+ branch |
+ last updated |
+ build warnings* |
+ lint failures |
+
+
+
+{% for project, metadata in metadata | sort(attribute='0.completion,0.translators.number') | reverse %}
+
+ | {{ project.language.name }} ({{ project.language.code }}) |
+ {{ project.branch }} |
+ {{ metadata[2].strftime('%Y/%m/%d %T') if metadata[2] else '' }} |
+
+ {% if project.completion %}{{ metadata[0] }}{% else %}{{ metadata[0] }}{% endif %}
+ |
+
+ {% if project.completion %}{{ metadata[1] }}{% else %}{{ metadata[1] }}{% endif %}
+ |
+
+{% endfor %}
+
+
+* number of Sphinx build process warnings
+For more information about translations, see the Python Developer’s Guide.
+Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).
+
+
diff --git a/sphinx_lint.py b/sphinx_lint.py
new file mode 100644
index 000000000..167cec1b4
--- /dev/null
+++ b/sphinx_lint.py
@@ -0,0 +1,18 @@
+from collections.abc import Iterator
+from itertools import chain
+from pathlib import Path
+
+from sphinxlint import check_file, checkers
+
+
+def store_and_count_failures(clones_dir: str, repo: str, language_code: str) -> int:
+ failed_checks = list(chain.from_iterable(yield_failures(clones_dir, repo)))
+ filepath = Path(f'warnings-lint-{language_code}.txt')
+ filepath.write_text('\n'.join([str(c) for c in failed_checks]))
+ return len(failed_checks)
+
+
+def yield_failures(clones_dir: str, repo: str) -> Iterator[str]:
+ enabled_checkers = [c for c in checkers.all_checkers.values() if c.enabled]
+ for path in Path(clones_dir, 'rebased_translations', repo).rglob('*.po'):
+ yield check_file(path.as_posix(), enabled_checkers)
diff --git a/style.css b/style.css
index 86e90baff..18e3eaf12 100644
--- a/style.css
+++ b/style.css
@@ -3,7 +3,6 @@ body {
}
table {
border-collapse: collapse;
- width: 100%;
}
th, td {
border: 1px solid #ddd;
@@ -38,13 +37,25 @@ th {
.progress-bar.low + .progress-bar-outer-label {
display: inline-block;
}
-td[data-label="translators"] {
+td[data-label="translators"], td[data-label="warnings"], td[data-label="lint"] {
text-align: right;
}
td[data-label="completion"] {
width: 100%;
line-height: 0;
}
+.switchpages{
+ position: absolute;
+ top: 10px;
+ right: 10px;
+ }
+
+@media screen and (max-width: 675px) {
+ .switchpages{
+ all: unset;
+ }
+}
+
@media screen and (max-width: 600px) {
table, thead, tbody, th, td, tr {
display: block;
diff --git a/template.html.jinja b/template.html.jinja
index 92f3446e5..d1432b2b5 100644
--- a/template.html.jinja
+++ b/template.html.jinja
@@ -4,9 +4,13 @@
+
Python Docs Translation Dashboard
+
* the number in parentheses shows change in the last 30 days, included in the total completion
-For more information about translations, see the Python Developer’s Guide.
+For more information about translations, see the Python Developer’s Guide.
Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).