From be7fc31383852dbdc621b7fa32c4f0581337744e Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sun, 12 Jan 2025 22:32:24 +0100 Subject: [PATCH 01/20] Move template to a separate file --- generate.py | 60 ++------------------------------------------------- template.html | 52 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 58 deletions(-) create mode 100644 template.html diff --git a/generate.py b/generate.py index 82390a6ab..623e72b28 100644 --- a/generate.py +++ b/generate.py @@ -61,66 +61,10 @@ ) print(completion_progress[-1]) -template = Template( - """ - - - Python Docs Translation Dashboard - - - -

Python Docs Translation Dashboard

- - - - - - - - - - - -{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %} - - {% if repo %} - - {% else %} - - {% endif %} - - - - - -{% endfor %} - -
languagebuildvisitorstranslatorscompletion
- - {{ language }} - - {{ language }} - {% if build %} - ✓{% if in_switcher %} in switcher{% endif %} - {% else %} - ✗ - {% endif %} - - - {{ '{:,}'.format(visitors) }} - - {{ '{:,}'.format(translators) }} -
{{ completion | round(2) }}%
-
-

Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.

- - -""" -) +template = Template(Path('template.html').read_text()) output = template.render( completion_progress=completion_progress, generation_time=generation_time ) -with open('index.html', 'w') as file: - file.write(output) +Path('index.html').write_text(output) diff --git a/template.html b/template.html new file mode 100644 index 000000000..09441b9af --- /dev/null +++ b/template.html @@ -0,0 +1,52 @@ + + + Python Docs Translation Dashboard + + + +

Python Docs Translation Dashboard

+ + + + + + + + + + + +{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %} + + {% if repo %} + + {% else %} + + {% endif %} + + + + + +{% endfor %} + +
languagebuildvisitorstranslatorscompletion
+ + {{ language }} + + {{ language }} + {% if build %} + ✓{% if in_switcher %} in switcher{% endif %} + {% else %} + ✗ + {% endif %} + + + {{ '{:,}'.format(visitors) }} + + {{ translators }} +
{{ completion | round(2) }}%
+
+

Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.

+ + From 5c689ae463ec7ca09c41a280323d716f055bb8cc Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sun, 12 Jan 2025 22:35:00 +0100 Subject: [PATCH 02/20] Get number of visitors for only built langs --- generate.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/generate.py b/generate.py index 623e72b28..9e1552706 100644 --- a/generate.py +++ b/generate.py @@ -43,9 +43,13 @@ ) languages_built = dict(build_status.get_languages()) for language, repo in repositories.get_languages_and_repos(devguide_dir): + built_on_docs_python_org = language in languages_built if repo: completion_number, translators_number = get_completion(clones_dir, repo) - visitors_number = visitors.get_number_of_visitors(language) + if built_on_docs_python_org: + visitors_number = visitors.get_number_of_visitors(language) + else: + visitors_number = 0 else: completion_number, translators_number, visitors_number = 0.0, 0, 0 completion_progress.append( @@ -55,7 +59,7 @@ completion_number, translators_number, visitors_number, - language in languages_built, # built on docs.python.org + built_on_docs_python_org, languages_built.get(language), # in switcher ) ) From 869dc176033747422ca605f14eab4bf08f195ddb Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sun, 12 Jan 2025 22:36:13 +0100 Subject: [PATCH 03/20] Raise timeout for getting number of visitors It used to break the CI. --- visitors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visitors.py b/visitors.py index fde5d593e..07399f15a 100644 --- a/visitors.py +++ b/visitors.py @@ -10,7 +10,7 @@ def get_number_of_visitors(language: str) -> int: param = urllib.parse.urlencode( {'filters': f'[["contains","event:page",["/{language}/"]]]'} ) - r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=10) + r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=20) with ( zipfile.ZipFile(io.BytesIO(r.content), 'r') as z, z.open('visitors.csv') as csv_file, From 745b92435e0439e2f6df963d1ed18753bec20950 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sun, 12 Jan 2025 22:53:03 +0100 Subject: [PATCH 04/20] Convert main loop into iterator --- generate.py | 63 +++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/generate.py b/generate.py index 9e1552706..38ef19184 100644 --- a/generate.py +++ b/generate.py @@ -9,6 +9,7 @@ # ] # /// import subprocess +from collections.abc import Iterator from datetime import datetime, timezone from pathlib import Path from tempfile import TemporaryDirectory @@ -24,51 +25,55 @@ completion_progress = [] generation_time = datetime.now(timezone.utc) -with TemporaryDirectory() as clones_dir: - Repo.clone_from( - 'https://github.com/python/devguide.git', - devguide_dir := Path(clones_dir, 'devguide'), - depth=1, - ) - latest_branch = branches_from_devguide(devguide_dir)[0] - Repo.clone_from( - 'https://github.com/python/cpython.git', - Path(clones_dir, 'cpython'), - depth=1, - branch=latest_branch, - ) - subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True) - subprocess.run( - ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True - ) - languages_built = dict(build_status.get_languages()) - for language, repo in repositories.get_languages_and_repos(devguide_dir): - built_on_docs_python_org = language in languages_built - if repo: + +def get_completion_progress() -> ( + Iterator[tuple[str, str, float, int, int, bool, bool | None]] +): + with TemporaryDirectory() as clones_dir: + Repo.clone_from( + 'https://github.com/python/devguide.git', + devguide_dir := Path(clones_dir, 'devguide'), + depth=1, + ) + latest_branch = branches_from_devguide(devguide_dir)[0] + Repo.clone_from( + 'https://github.com/python/cpython.git', + Path(clones_dir, 'cpython'), + depth=1, + branch=latest_branch, + ) + subprocess.run( + ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True + ) + subprocess.run( + ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True + ) + languages_built = dict(build_status.get_languages()) + for language, repo in repositories.get_languages_and_repos(devguide_dir): + built_on_docs_python_org = language in languages_built + in_switcher = languages_built.get(language) + if not repo: + yield language, repo, 0.0, 0, 0, built_on_docs_python_org, in_switcher completion_number, translators_number = get_completion(clones_dir, repo) if built_on_docs_python_org: visitors_number = visitors.get_number_of_visitors(language) else: visitors_number = 0 - else: - completion_number, translators_number, visitors_number = 0.0, 0, 0 - completion_progress.append( - ( + yield ( language, repo, completion_number, translators_number, visitors_number, built_on_docs_python_org, - languages_built.get(language), # in switcher + in_switcher, ) - ) - print(completion_progress[-1]) + template = Template(Path('template.html').read_text()) output = template.render( - completion_progress=completion_progress, generation_time=generation_time + completion_progress=get_completion_progress(), generation_time=generation_time ) Path('index.html').write_text(output) From d0b1b761b24ecc885b484c92cfa171897cefe81a Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sun, 12 Jan 2025 23:47:58 +0100 Subject: [PATCH 05/20] Convert context to tuple --- generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate.py b/generate.py index 38ef19184..75a0b4c7f 100644 --- a/generate.py +++ b/generate.py @@ -73,7 +73,7 @@ def get_completion_progress() -> ( template = Template(Path('template.html').read_text()) output = template.render( - completion_progress=get_completion_progress(), generation_time=generation_time + completion_progress=tuple(get_completion_progress()), generation_time=generation_time ) Path('index.html').write_text(output) From 68227132caf1c8a35cf898ce2dd289117f46dd15 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 12 Jan 2025 22:48:04 +0000 Subject: [PATCH 06/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generate.py b/generate.py index 75a0b4c7f..afac1c95a 100644 --- a/generate.py +++ b/generate.py @@ -73,7 +73,8 @@ def get_completion_progress() -> ( template = Template(Path('template.html').read_text()) output = template.render( - completion_progress=tuple(get_completion_progress()), generation_time=generation_time + completion_progress=tuple(get_completion_progress()), + generation_time=generation_time, ) Path('index.html').write_text(output) From 4a22dca850a9b471eac06ecf51c903c86d89c76d Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 00:46:41 +0100 Subject: [PATCH 07/20] Iterator: continue when no repo --- generate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/generate.py b/generate.py index afac1c95a..2c4005601 100644 --- a/generate.py +++ b/generate.py @@ -54,6 +54,7 @@ def get_completion_progress() -> ( in_switcher = languages_built.get(language) if not repo: yield language, repo, 0.0, 0, 0, built_on_docs_python_org, in_switcher + continue completion_number, translators_number = get_completion(clones_dir, repo) if built_on_docs_python_org: visitors_number = visitors.get_number_of_visitors(language) From 00279a4ef6a5d9d4c15daac252de6e8a79022c89 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 00:48:07 +0100 Subject: [PATCH 08/20] Add main block to generate module, revert tuple conversion --- generate.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/generate.py b/generate.py index 2c4005601..607b1afe8 100644 --- a/generate.py +++ b/generate.py @@ -71,11 +71,12 @@ def get_completion_progress() -> ( ) -template = Template(Path('template.html').read_text()) +if __name__ == '__main__': + template = Template(Path('template.html').read_text()) -output = template.render( - completion_progress=tuple(get_completion_progress()), - generation_time=generation_time, -) + output = template.render( + completion_progress=get_completion_progress(), + generation_time=generation_time, + ) -Path('index.html').write_text(output) + Path('index.html').write_text(output) From ae763dc39fdabaa5e1e30d17ae90f0428f0c42fe Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 00:48:41 +0100 Subject: [PATCH 09/20] Don't link to Plausible when no build --- template.html | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/template.html b/template.html index 09441b9af..c01b516c0 100644 --- a/template.html +++ b/template.html @@ -35,9 +35,13 @@

Python Docs Translation Dashboard

{% endif %} - - {{ '{:,}'.format(visitors) }} - + {% if build %} + + {{ '{:,}'.format(visitors) }} + + {% else %} + 0 + {% endif %} {{ translators }} From a1b977bd0f3029cb1203d3f0100f5e8e64008aa6 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 01:19:24 +0100 Subject: [PATCH 10/20] Add mypy --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1214a8ee9..6c3caec03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,5 +34,10 @@ repos: - id: check-hooks-apply - id: check-useless-excludes + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.14.1 + hooks: + - id: mypy + ci: autoupdate_schedule: quarterly From 9a96bf2ad7e0cedb2a87c6d66166dea0188f5e75 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 01:23:27 +0100 Subject: [PATCH 11/20] Add mypy's additional deps --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c3caec03..62ea33448 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,6 +38,7 @@ repos: rev: v1.14.1 hooks: - id: mypy + additional_dependencies: [types-polib, types-docutils, types-requests] ci: autoupdate_schedule: quarterly From 8eba337f271ee06f21fdeb953d678b922f318e71 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 01:45:03 +0100 Subject: [PATCH 12/20] Types improvements --- generate.py | 12 ++++++++++-- repositories.py | 13 ++++++++----- translators.py | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/generate.py b/generate.py index 607b1afe8..bcffff7ea 100644 --- a/generate.py +++ b/generate.py @@ -13,6 +13,7 @@ from datetime import datetime, timezone from pathlib import Path from tempfile import TemporaryDirectory +from typing import cast from git import Repo from jinja2 import Template @@ -22,7 +23,6 @@ import visitors from completion import branches_from_devguide, get_completion -completion_progress = [] generation_time = datetime.now(timezone.utc) @@ -53,7 +53,15 @@ def get_completion_progress() -> ( built_on_docs_python_org = language in languages_built in_switcher = languages_built.get(language) if not repo: - yield language, repo, 0.0, 0, 0, built_on_docs_python_org, in_switcher + yield ( + language, + cast(str, repo), + 0.0, + 0, + 0, + built_on_docs_python_org, + in_switcher, + ) continue completion_number, translators_number = get_completion(clones_dir, repo) if built_on_docs_python_org: diff --git a/repositories.py b/repositories.py index c266a8be1..dd82b8357 100644 --- a/repositories.py +++ b/repositories.py @@ -1,6 +1,6 @@ import pathlib import re -from typing import Generator, Optional +from collections.abc import Iterator from docutils import core from docutils.nodes import table, row @@ -8,7 +8,7 @@ def get_languages_and_repos( devguide_dir: pathlib.Path, -) -> Generator[tuple[str, Optional[str]], None, None]: +) -> Iterator[tuple[str, str | None]]: translating = devguide_dir.joinpath('documentation/translating.rst').read_text() doctree = core.publish_doctree(translating) @@ -16,8 +16,11 @@ def get_languages_and_repos( for row_node in node.traverse(row)[1:]: language = row_node[0].astext() repo = row_node[2].astext() - language_code = ( - re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-') - ) + language_match = re.match(r'.* \((.*)\)', language) + if not language_match: + raise ValueError( + f'Expected a language code in brackets in devguide table, found {language}' + ) + language_code = language_match.group(1).lower().replace('_', '-') repo_match = re.match(':github:`GitHub <(.*)>`', repo) yield language_code, repo_match and repo_match.group(1) diff --git a/translators.py b/translators.py index 2fe11ccbc..0ef324b16 100644 --- a/translators.py +++ b/translators.py @@ -18,7 +18,7 @@ def get_number_from_git_history(path: Path) -> int: def yield_from_headers(path: Path) -> Generator[str, None, None]: for file in path.rglob('*.po'): try: - header = pofile(file).header.splitlines() + header = pofile(file).header.splitlines() # type: ignore[call-overload] # https://github.com/python/typeshed/pull/13396 except IOError: continue if 'Translators:' not in header: From c3f60e3b8a69a48f41a214a2b92e963de5191eab Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 01:49:08 +0100 Subject: [PATCH 13/20] ruff format skip-magic-trailing-comma --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index 749ef043d..9b30b0744 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,2 +1,3 @@ [format] quote-style = "single" +skip-magic-trailing-comma = true From 4de85d3c0f0d36d6893efef124766239ea619c16 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 01:50:16 +0100 Subject: [PATCH 14/20] reformat --- repositories.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/repositories.py b/repositories.py index dd82b8357..2ab70ec32 100644 --- a/repositories.py +++ b/repositories.py @@ -1,14 +1,12 @@ -import pathlib import re from collections.abc import Iterator +from pathlib import Path from docutils import core from docutils.nodes import table, row -def get_languages_and_repos( - devguide_dir: pathlib.Path, -) -> Iterator[tuple[str, str | None]]: +def get_languages_and_repos(devguide_dir: Path) -> Iterator[tuple[str, str | None]]: translating = devguide_dir.joinpath('documentation/translating.rst').read_text() doctree = core.publish_doctree(translating) From ad9ebee591e23b312a54d31c77d6c288d1eeab4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 01:29:51 +0000 Subject: [PATCH 15/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- generate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generate.py b/generate.py index bcffff7ea..425e7534f 100644 --- a/generate.py +++ b/generate.py @@ -83,8 +83,7 @@ def get_completion_progress() -> ( template = Template(Path('template.html').read_text()) output = template.render( - completion_progress=get_completion_progress(), - generation_time=generation_time, + completion_progress=get_completion_progress(), generation_time=generation_time ) Path('index.html').write_text(output) From 07e3e3fb824ee2ff79ec766e81379301abce2fa0 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 09:02:18 +0100 Subject: [PATCH 16/20] improve type --- translators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/translators.py b/translators.py index 0ef324b16..79fa57f35 100644 --- a/translators.py +++ b/translators.py @@ -1,4 +1,4 @@ -from collections.abc import Generator +from collections.abc import Iterator from pathlib import Path from git import Repo @@ -15,7 +15,7 @@ def get_number_from_git_history(path: Path) -> int: return len(Repo(path).git.shortlog('-s', 'HEAD').splitlines()) -def yield_from_headers(path: Path) -> Generator[str, None, None]: +def yield_from_headers(path: Path) -> Iterator[str]: for file in path.rglob('*.po'): try: header = pofile(file).header.splitlines() # type: ignore[call-overload] # https://github.com/python/typeshed/pull/13396 From bcd598dce22eb8e7a4d08dff5bc43d986439c466 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 09:16:16 +0100 Subject: [PATCH 17/20] Rename the template --- generate.py | 2 +- template.html => template.html.jinja | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename template.html => template.html.jinja (100%) diff --git a/generate.py b/generate.py index 425e7534f..4d05ba45f 100644 --- a/generate.py +++ b/generate.py @@ -80,7 +80,7 @@ def get_completion_progress() -> ( if __name__ == '__main__': - template = Template(Path('template.html').read_text()) + template = Template(Path('template.html.jinja').read_text()) output = template.render( completion_progress=get_completion_progress(), generation_time=generation_time diff --git a/template.html b/template.html.jinja similarity index 100% rename from template.html rename to template.html.jinja From 147cd053dcfdfa8a7b8a5de552b85dfbbe077857 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 09:37:56 +0100 Subject: [PATCH 18/20] Rename variables --- generate.py | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/generate.py b/generate.py index 4d05ba45f..f273e7489 100644 --- a/generate.py +++ b/generate.py @@ -49,34 +49,15 @@ def get_completion_progress() -> ( ['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True ) languages_built = dict(build_status.get_languages()) - for language, repo in repositories.get_languages_and_repos(devguide_dir): - built_on_docs_python_org = language in languages_built - in_switcher = languages_built.get(language) + for lang, repo in repositories.get_languages_and_repos(devguide_dir): + built = lang in languages_built + in_switcher = languages_built.get(lang) if not repo: - yield ( - language, - cast(str, repo), - 0.0, - 0, - 0, - built_on_docs_python_org, - in_switcher, - ) + yield lang, cast(str, repo), 0.0, 0, 0, built, in_switcher continue - completion_number, translators_number = get_completion(clones_dir, repo) - if built_on_docs_python_org: - visitors_number = visitors.get_number_of_visitors(language) - else: - visitors_number = 0 - yield ( - language, - repo, - completion_number, - translators_number, - visitors_number, - built_on_docs_python_org, - in_switcher, - ) + completion, translators = get_completion(clones_dir, repo) + visitors_num = visitors.get_number_of_visitors(lang) if built else 0 + yield lang, repo, completion, translators, visitors_num, built, in_switcher if __name__ == '__main__': From 14fb926a610b1f34caac2ffee43773efd25b88e2 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 09:40:46 +0100 Subject: [PATCH 19/20] open visitors header link as a blank page --- template.html.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.html.jinja b/template.html.jinja index c01b516c0..91a0c176d 100644 --- a/template.html.jinja +++ b/template.html.jinja @@ -10,7 +10,7 @@ language build - visitors + visitors translators completion From 9d46636c143501474329e3c3c24707c7919e4f0e Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Jan 2025 09:41:38 +0100 Subject: [PATCH 20/20] order mypy deps alphabetically --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62ea33448..043d312a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: rev: v1.14.1 hooks: - id: mypy - additional_dependencies: [types-polib, types-docutils, types-requests] + additional_dependencies: [types-docutils, types-polib, types-requests] ci: autoupdate_schedule: quarterly