Skip to content

Commit 3eb3a09

Browse files
m-aciekm-aciek
authored
Metadata (#58)
1 parent 02c2124 commit 3eb3a09

File tree

10 files changed

+274
-51
lines changed

10 files changed

+274
-51
lines changed

‎.github/workflows/update.yml‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
on:
22
schedule:
3-
- cron: '*/30 * * * *'
3+
- cron: '17 * * * *'
44
push:
55
branches:
66
- 'main'
@@ -19,7 +19,7 @@ jobs:
1919
- uses: astral-sh/setup-uv@v5
2020
- uses: actions/checkout@v4
2121
- run: sudo apt-get install -y gettext
22-
- run: uv run generate.py # generates "index.html"
22+
- run: uv run generate.py # generates index.html and index.json
2323
- run: mkdir -p build && cp index.* style.css build
2424
- name: Deploy 🚀
2525
if: github.event_name != 'pull_request'
@@ -53,9 +53,28 @@ jobs:
5353
- name: Debug index.html if pull request
5454
if: github.event_name == 'pull_request'
5555
run: |
56-
curl -Lo index.html-public https://github.com/m-aciek/pydocs-translation-dashboard/raw/refs/heads/gh-pages/index.html
56+
curl -Lo index.html-public https://github.com/python-docs-translations/dashboard/raw/refs/heads/gh-pages/index.html
5757
diff --color=always -u index.html-public index.html || :
5858
cat index.html
59+
- run: uv run generate_metadata.py # generates metadata.html
60+
- run: cp metadata.html warnings* build
61+
- name: Deploy metadata view 🚀
62+
if: github.event_name != 'pull_request'
63+
uses: JamesIves/github-pages-deploy-action@v4
64+
with:
65+
folder: build
66+
clean: false
67+
git-config-name: github-actions[bot]
68+
git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
69+
- name: Deploy metadata view to subdirectory if pull request 🚀
70+
if: github.event_name == 'pull_request'
71+
uses: JamesIves/github-pages-deploy-action@v4
72+
with:
73+
folder: build
74+
target-folder: ${{ github.ref_name }}
75+
clean: false
76+
git-config-name: github-actions[bot]
77+
git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
5978
- uses: actions/upload-artifact@v4
6079
with:
6180
name: build

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
11
index.html
2+
metadata.html
3+
warnings-*.txt
4+
clones

‎build_warnings.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
from pathlib import Path
2+
from re import findall
3+
from shutil import copyfile
4+
5+
import sphinx.cmd.build
6+
7+
8+
def number(clones_dir: str, repo: str, language_code: str) -> int:
9+
language_part, *locale = language_code.split('-')
10+
if locale:
11+
lang_with_locale = f'{language_part}_{locale[0].upper()}'
12+
else:
13+
lang_with_locale = language_part
14+
locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES')
15+
locale_dir.mkdir(parents=True, exist_ok=True)
16+
for po_file in (repo_dir := Path(clones_dir, 'translations', repo)).rglob('*.po'):
17+
relative_path = po_file.relative_to(repo_dir)
18+
target_file = locale_dir / relative_path
19+
target_file.parent.mkdir(parents=True, exist_ok=True)
20+
copyfile(po_file, target_file)
21+
sphinx.cmd.build.main(
22+
(
23+
'--builder',
24+
'html',
25+
'--jobs',
26+
'auto',
27+
'--define',
28+
f'language={language_code}',
29+
'--verbose',
30+
'--warning-file',
31+
warning_file := f'{clones_dir}/warnings-{language_code}.txt',
32+
f'{clones_dir}/cpython/Doc', # sourcedir
33+
f'./sphinxbuild/{language_code}', # outputdir
34+
)
35+
)
36+
copyfile(warning_file, f'warnings-{language_code}.txt')
37+
return len(findall('ERROR|WARNING', Path(warning_file).read_text()))

‎completion.py‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,32 +23,36 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]:
2323
def get_completion(
2424
clones_dir: str, repo: str
2525
) -> tuple[float, 'TranslatorsData', str, float]:
26-
clone_path = Path(clones_dir, repo)
27-
for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
26+
clone_path = Path(clones_dir, 'translations', repo)
27+
for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + [
28+
'master',
29+
'main',
30+
]:
2831
try:
2932
clone_repo = git.Repo.clone_from(
3033
f'https://github.com/{repo}.git', clone_path, branch=branch
3134
)
3235
except git.GitCommandError:
3336
print(f'failed to clone {repo} {branch}')
3437
translators_data = TranslatorsData(0, False)
38+
branch = ''
3539
continue
3640
else:
3741
translators_number = translators.get_number(clone_path)
3842
translators_link = translators.get_link(clone_path, repo, branch)
3943
translators_data = TranslatorsData(translators_number, translators_link)
4044
break
41-
with TemporaryDirectory() as tmpdir:
42-
completion = potodo.merge_and_scan_path(
43-
clone_path,
44-
pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
45-
merge_path=Path(tmpdir),
46-
hide_reserved=False,
47-
api_url='',
48-
).completion
45+
path_for_merge = Path(clones_dir, 'rebased_translations', repo)
46+
completion = potodo.merge_and_scan_path(
47+
clone_path,
48+
pot_path=Path(clones_dir, 'cpython/Doc/build/gettext'),
49+
merge_path=path_for_merge,
50+
hide_reserved=False,
51+
api_url='',
52+
).completion
4953

5054
if completion:
51-
# Fetch commit from before 30 days ago and checkout
55+
# Get latest commit date and fetch commit from before 30 days ago and checkout
5256
try:
5357
commit = next(
5458
clone_repo.iter_commits('HEAD', max_count=1, before='30 days ago')

‎generate.py‎

Lines changed: 27 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -16,54 +16,51 @@
1616
from dataclasses import dataclass, asdict
1717
from datetime import datetime, timezone
1818
from pathlib import Path
19-
from tempfile import TemporaryDirectory
2019

2120
from git import Repo
2221
from jinja2 import Template
2322
from urllib3 import PoolManager
2423

25-
import contribute
2624
import build_status
25+
import contribute
2726
from completion import branches_from_devguide, get_completion, TranslatorsData
28-
from repositories import get_languages_and_repos, Language
27+
from repositories import Language, get_languages_and_repos
2928

3029
generation_time = datetime.now(timezone.utc)
3130

3231

3332
def get_completion_progress() -> Iterator['LanguageProjectData']:
34-
with TemporaryDirectory() as clones_dir:
35-
Repo.clone_from(
36-
'https://github.com/python/devguide.git',
37-
devguide_dir := Path(clones_dir, 'devguide'),
38-
depth=1,
39-
)
40-
latest_branch = branches_from_devguide(devguide_dir)[0]
41-
Repo.clone_from(
42-
'https://github.com/python/cpython.git',
43-
cpython_dir := Path(clones_dir, 'cpython'),
44-
depth=1,
45-
branch=latest_branch,
46-
)
47-
subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
48-
subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
49-
languages_built = dict(build_status.get_languages(http := PoolManager()))
33+
clones_dir = Path('clones')
34+
Repo.clone_from(
35+
'https://github.com/python/devguide.git',
36+
devguide_dir := Path(clones_dir, 'devguide'),
37+
depth=1,
38+
)
39+
latest_branch = branches_from_devguide(devguide_dir)[0]
40+
Repo.clone_from(
41+
'https://github.com/python/cpython.git',
42+
cpython_dir := Path(clones_dir, 'cpython'),
43+
depth=1,
44+
branch=latest_branch,
45+
)
46+
subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
47+
subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
48+
languages_built = dict(build_status.get_languages(PoolManager()))
5049

51-
with concurrent.futures.ThreadPoolExecutor() as executor:
52-
return executor.map(
53-
get_project_data,
54-
*zip(*get_languages_and_repos(devguide_dir)),
55-
itertools.repeat(languages_built),
56-
itertools.repeat(clones_dir),
57-
itertools.repeat(http),
58-
)
50+
with concurrent.futures.ThreadPoolExecutor() as executor:
51+
return executor.map(
52+
get_project_data,
53+
*zip(*get_languages_and_repos(devguide_dir)),
54+
itertools.repeat(languages_built),
55+
itertools.repeat(clones_dir),
56+
)
5957

6058

6159
def get_project_data(
6260
language: Language,
6361
repo: str | None,
6462
languages_built: dict[str, bool],
6563
clones_dir: str,
66-
http: PoolManager,
6764
) -> 'LanguageProjectData':
6865
built = language.code in languages_built
6966
if repo:
@@ -72,7 +69,7 @@ def get_project_data(
7269
completion = 0.0
7370
translators_data = TranslatorsData(0, False)
7471
change = 0.0
75-
branch = None
72+
branch = ''
7673
return LanguageProjectData(
7774
language,
7875
repo,
@@ -91,7 +88,7 @@ def get_project_data(
9188
class LanguageProjectData:
9289
language: Language
9390
repository: str | None
94-
branch: str | None
91+
branch: str
9592
completion: float
9693
change: float
9794
translators: TranslatorsData

‎generate_metadata.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# /// script
2+
# requires-python = ">=3.11"
3+
# dependencies = [
4+
# "gitpython",
5+
# "potodo",
6+
# "jinja2",
7+
# "sphinx",
8+
# "python-docs-theme",
9+
# "dacite",
10+
# "sphinx-lint",
11+
# ]
12+
# ///
13+
import concurrent.futures
14+
import itertools
15+
import logging
16+
from collections.abc import Iterator, Sequence
17+
from datetime import datetime, timezone
18+
from json import loads
19+
from pathlib import Path
20+
from sys import argv
21+
22+
import dacite
23+
from git import Repo
24+
from jinja2 import Template
25+
from urllib3 import request
26+
27+
import build_warnings
28+
import sphinx_lint
29+
from generate import LanguageProjectData
30+
from repositories import Language
31+
32+
generation_time = datetime.now(timezone.utc)
33+
34+
35+
def get_projects_metadata(
36+
completion_progress: Sequence[LanguageProjectData],
37+
) -> Iterator[tuple[int, int, datetime | None]]:
38+
with concurrent.futures.ProcessPoolExecutor() as executor:
39+
return executor.map(
40+
get_metadata,
41+
*zip(*map(get_language_repo_and_completion, completion_progress)),
42+
itertools.repeat(Path('clones')),
43+
)
44+
45+
46+
def get_metadata(
47+
language: Language, repo: str | None, completion: float, clones_dir: str
48+
) -> tuple[int, int, datetime | None]:
49+
if not repo or not (repo_path := Path(clones_dir, 'translations', repo)).exists():
50+
return 0, 0, None
51+
(clone_repo := Repo(repo_path)).git.checkout()
52+
latest_commit = clone_repo.head.commit.committed_datetime
53+
if not completion:
54+
return 0, 0, latest_commit
55+
return (
56+
build_warnings.number(clones_dir, repo, language.code),
57+
sphinx_lint.store_and_count_failures(clones_dir, repo, language.code),
58+
latest_commit,
59+
)
60+
61+
62+
def get_language_repo_and_completion(
63+
project: LanguageProjectData,
64+
) -> tuple[Language, str | None, float]:
65+
return project.language, project.repository, project.completion
66+
67+
68+
if __name__ == '__main__':
69+
logging.basicConfig(level=logging.INFO)
70+
logging.info(f'starting at {generation_time}')
71+
template = Template(Path('metadata.html.jinja').read_text())
72+
if (index_path := Path('index.json')).exists():
73+
index_json = loads(Path('index.json').read_text())
74+
else:
75+
index_json = request('GET', argv[1]).json()
76+
77+
completion_progress = [
78+
dacite.from_dict(LanguageProjectData, project) for project in index_json
79+
]
80+
81+
output = template.render(
82+
metadata=zip(completion_progress, get_projects_metadata(completion_progress)),
83+
generation_time=generation_time,
84+
duration=(datetime.now(timezone.utc) - generation_time).seconds,
85+
)
86+
87+
Path('metadata.html').write_text(output)

‎metadata.html.jinja‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<html lang="en">
2+
<head>
3+
<title>Python Docs Translation Dashboard</title>
4+
<link rel="stylesheet" href="style.css">
5+
<meta charset="UTF-8">
6+
<base target="_blank">
7+
</head>
8+
<body>
9+
<h1>Python Docs Translation Dashboard</h1>
10+
<nav class="switchpages">
11+
<a href="index.html" target="_self">main</a> | meta
12+
</nav>
13+
<table>
14+
<thead>
15+
<tr>
16+
<th>language</th>
17+
<th>branch</th>
18+
<th>last updated</th>
19+
<th>build warnings*</th>
20+
<th>lint failures</th>
21+
</tr>
22+
</thead>
23+
<tbody>
24+
{% for project, metadata in metadata | sort(attribute='0.completion,0.translators.number') | reverse %}
25+
<tr>
26+
<td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
27+
<td data-label="branch">{{ project.branch }}</td>
28+
<td data-label="updated">{{ metadata[2].strftime('%Y/%m/%d %T') if metadata[2] else '' }}</td>
29+
<td data-label="warnings">
30+
{% if project.completion %}<a href="warnings-{{ project.language.code }}.txt">{{ metadata[0] }}</a>{% else %}{{ metadata[0] }}{% endif %}
31+
</td>
32+
<td data-label="lint">
33+
{% if project.completion %}<a href="warnings-lint-{{ project.language.code }}.txt">{{ metadata[1] }}</a>{% else %}{{ metadata[1] }}{% endif %}
34+
</td>
35+
</tr>
36+
{% endfor %}
37+
</tbody>
38+
</table>
39+
<p>* number of Sphinx build process warnings</p>
40+
<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/">Python Developer’s Guide</a>.</p>
41+
<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
42+
</body>
43+
</html>

‎sphinx_lint.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from collections.abc import Iterator
2+
from itertools import chain
3+
from pathlib import Path
4+
5+
from sphinxlint import check_file, checkers
6+
7+
8+
def store_and_count_failures(clones_dir: str, repo: str, language_code: str) -> int:
9+
failed_checks = list(chain.from_iterable(yield_failures(clones_dir, repo)))
10+
filepath = Path(f'warnings-lint-{language_code}.txt')
11+
filepath.write_text('\n'.join([str(c) for c in failed_checks]))
12+
return len(failed_checks)
13+
14+
15+
def yield_failures(clones_dir: str, repo: str) -> Iterator[str]:
16+
enabled_checkers = [c for c in checkers.all_checkers.values() if c.enabled]
17+
for path in Path(clones_dir, 'rebased_translations', repo).rglob('*.po'):
18+
yield check_file(path.as_posix(), enabled_checkers)

0 commit comments

Comments
 (0)