diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1214a8ee9..043d312a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,5 +34,11 @@ repos: - id: check-hooks-apply - id: check-useless-excludes + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.14.1 + hooks: + - id: mypy + additional_dependencies: [types-docutils, types-polib, types-requests] + ci: autoupdate_schedule: quarterly diff --git a/generate.py b/generate.py index 82390a6ab..f273e7489 100644 --- a/generate.py +++ b/generate.py @@ -9,9 +9,11 @@ # ] # /// import subprocess +from collections.abc import Iterator from datetime import datetime, timezone from pathlib import Path from tempfile import TemporaryDirectory +from typing import cast from git import Repo from jinja2 import Template @@ -21,106 +23,48 @@ import visitors from completion import branches_from_devguide, get_completion -completion_progress = [] generation_time = datetime.now(timezone.utc) -with TemporaryDirectory() as clones_dir: - Repo.clone_from( - 'https://github.com/python/devguide.git', - devguide_dir := Path(clones_dir, 'devguide'), - depth=1, - ) - latest_branch = branches_from_devguide(devguide_dir)[0] - Repo.clone_from( - 'https://github.com/python/cpython.git', - Path(clones_dir, 'cpython'), - depth=1, - branch=latest_branch, - ) - subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True) - subprocess.run( - ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True - ) - languages_built = dict(build_status.get_languages()) - for language, repo in repositories.get_languages_and_repos(devguide_dir): - if repo: - completion_number, translators_number = get_completion(clones_dir, repo) - visitors_number = visitors.get_number_of_visitors(language) - else: - completion_number, translators_number, visitors_number = 0.0, 0, 0 - completion_progress.append( - ( - language, - repo, - completion_number, - translators_number, - visitors_number, - language in languages_built, # built on docs.python.org - languages_built.get(language), # in switcher - ) + +def get_completion_progress() -> ( + Iterator[tuple[str, str, float, int, int, bool, bool | None]] +): + with TemporaryDirectory() as clones_dir: + Repo.clone_from( + 'https://github.com/python/devguide.git', + devguide_dir := Path(clones_dir, 'devguide'), + depth=1, + ) + latest_branch = branches_from_devguide(devguide_dir)[0] + Repo.clone_from( + 'https://github.com/python/cpython.git', + Path(clones_dir, 'cpython'), + depth=1, + branch=latest_branch, ) - print(completion_progress[-1]) + subprocess.run( + ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True + ) + subprocess.run( + ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True + ) + languages_built = dict(build_status.get_languages()) + for lang, repo in repositories.get_languages_and_repos(devguide_dir): + built = lang in languages_built + in_switcher = languages_built.get(lang) + if not repo: + yield lang, cast(str, repo), 0.0, 0, 0, built, in_switcher + continue + completion, translators = get_completion(clones_dir, repo) + visitors_num = visitors.get_number_of_visitors(lang) if built else 0 + yield lang, repo, completion, translators, visitors_num, built, in_switcher -template = Template( - """ - - - Python Docs Translation Dashboard - - - -

Python Docs Translation Dashboard

- - - - - - - - - - - -{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %} - - {% if repo %} - - {% else %} - - {% endif %} - - - - - -{% endfor %} - -
languagebuildvisitorstranslatorscompletion
- - {{ language }} - - {{ language }} - {% if build %} - ✓{% if in_switcher %} in switcher{% endif %} - {% else %} - ✗ - {% endif %} - - - {{ '{:,}'.format(visitors) }} - - {{ '{:,}'.format(translators) }} -
{{ completion | round(2) }}%
-
-

Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.

- - -""" -) -output = template.render( - completion_progress=completion_progress, generation_time=generation_time -) +if __name__ == '__main__': + template = Template(Path('template.html.jinja').read_text()) + + output = template.render( + completion_progress=get_completion_progress(), generation_time=generation_time + ) -with open('index.html', 'w') as file: - file.write(output) + Path('index.html').write_text(output) diff --git a/repositories.py b/repositories.py index c266a8be1..2ab70ec32 100644 --- a/repositories.py +++ b/repositories.py @@ -1,14 +1,12 @@ -import pathlib import re -from typing import Generator, Optional +from collections.abc import Iterator +from pathlib import Path from docutils import core from docutils.nodes import table, row -def get_languages_and_repos( - devguide_dir: pathlib.Path, -) -> Generator[tuple[str, Optional[str]], None, None]: +def get_languages_and_repos(devguide_dir: Path) -> Iterator[tuple[str, str | None]]: translating = devguide_dir.joinpath('documentation/translating.rst').read_text() doctree = core.publish_doctree(translating) @@ -16,8 +14,11 @@ def get_languages_and_repos( for row_node in node.traverse(row)[1:]: language = row_node[0].astext() repo = row_node[2].astext() - language_code = ( - re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-') - ) + language_match = re.match(r'.* \((.*)\)', language) + if not language_match: + raise ValueError( + f'Expected a language code in brackets in devguide table, found {language}' + ) + language_code = language_match.group(1).lower().replace('_', '-') repo_match = re.match(':github:`GitHub <(.*)>`', repo) yield language_code, repo_match and repo_match.group(1) diff --git a/ruff.toml b/ruff.toml index 749ef043d..9b30b0744 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,2 +1,3 @@ [format] quote-style = "single" +skip-magic-trailing-comma = true diff --git a/template.html.jinja b/template.html.jinja new file mode 100644 index 000000000..91a0c176d --- /dev/null +++ b/template.html.jinja @@ -0,0 +1,56 @@ + + + Python Docs Translation Dashboard + + + +

Python Docs Translation Dashboard

+ + + + + + + + + + + +{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %} + + {% if repo %} + + {% else %} + + {% endif %} + + + + + +{% endfor %} + +
languagebuildvisitorstranslatorscompletion
+ + {{ language }} + + {{ language }} + {% if build %} + ✓{% if in_switcher %} in switcher{% endif %} + {% else %} + ✗ + {% endif %} + + {% if build %} + + {{ '{:,}'.format(visitors) }} + + {% else %} + 0 + {% endif %} + {{ translators }} +
{{ completion | round(2) }}%
+
+

Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.

+ + diff --git a/translators.py b/translators.py index 2fe11ccbc..79fa57f35 100644 --- a/translators.py +++ b/translators.py @@ -1,4 +1,4 @@ -from collections.abc import Generator +from collections.abc import Iterator from pathlib import Path from git import Repo @@ -15,10 +15,10 @@ def get_number_from_git_history(path: Path) -> int: return len(Repo(path).git.shortlog('-s', 'HEAD').splitlines()) -def yield_from_headers(path: Path) -> Generator[str, None, None]: +def yield_from_headers(path: Path) -> Iterator[str]: for file in path.rglob('*.po'): try: - header = pofile(file).header.splitlines() + header = pofile(file).header.splitlines() # type: ignore[call-overload] # https://github.com/python/typeshed/pull/13396 except IOError: continue if 'Translators:' not in header: diff --git a/visitors.py b/visitors.py index fde5d593e..07399f15a 100644 --- a/visitors.py +++ b/visitors.py @@ -10,7 +10,7 @@ def get_number_of_visitors(language: str) -> int: param = urllib.parse.urlencode( {'filters': f'[["contains","event:page",["/{language}/"]]]'} ) - r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=10) + r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=20) with ( zipfile.ZipFile(io.BytesIO(r.content), 'r') as z, z.open('visitors.csv') as csv_file,