Merge branch 'main' into parallelize

Author: m-aciek

.github/workflows/update.yml

@@ -25,7 +25,7 @@ jobs:
         uses: JamesIves/github-pages-deploy-action@v4
         with:
           folder: build
-          clean: true
+          clean: false
           git-config-name: github-actions[bot]
           git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
       - name: Debug index.html if pull request

build_status.py

@@ -8,16 +8,15 @@
 import tomllib
 from collections.abc import Iterator
 
-from requests import Session
+from urllib3 import PoolManager
 
 
-def get_languages(requests: Session) -> Iterator[tuple[str, bool]]:
-    data = requests.get(
-        'https://raw.githubusercontent.com/'
-        'python/docsbuild-scripts/refs/heads/main/config.toml',
-        timeout=10,
-    ).text
-    config = tomllib.loads(data)
+def get_languages(http: PoolManager) -> Iterator[tuple[str, bool]]:
+    data = http.request(
+        'GET',
+        'https://raw.githubusercontent.com/python/docsbuild-scripts/refs/heads/main/config.toml',
+    ).data
+    config = tomllib.loads(data.decode())
     for code, language in config['languages'].items():
         language_code = code.lower().replace('_', '-')
         in_switcher = language.get('in_prod', config['defaults']['in_prod'])
@@ -26,7 +25,7 @@ def get_languages(requests: Session) -> Iterator[tuple[str, bool]]:
 
 def main() -> None:
     languages = {
-        language: in_switcher for language, in_switcher in get_languages(Session())
+        language: in_switcher for language, in_switcher in get_languages(PoolManager())
     }
     print(languages)
     for code in ('en', 'pl', 'ar', 'zh-cn', 'id'):

generate.py

@@ -14,13 +14,12 @@
 from collections.abc import Iterator
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from logging import info
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-from requests import Session
 from git import Repo
 from jinja2 import Template
+from urllib3 import PoolManager
 
 import contribute
 import build_status
@@ -47,14 +46,14 @@ def get_completion_progress() -> Iterator['LanguageProjectData']:
         )
         subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
         subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
-        languages_built = dict(build_status.get_languages(session := Session()))
+        languages_built = dict(build_status.get_languages(http := PoolManager()))
         with concurrent.futures.ThreadPoolExecutor() as executor:
             return executor.map(
                 get_project_data,
                 *zip(*get_languages_and_repos(devguide_dir)),
                 itertools.repeat(languages_built),
                 itertools.repeat(clones_dir),
-                itertools.repeat(session),
+                itertools.repeat(http),
             )
 
 
@@ -63,12 +62,12 @@ def get_project_data(
     repo: str,
     languages_built: dict[str, bool],
     clones_dir: str,
-    session: Session,
+    http: PoolManager,
 ) -> 'LanguageProjectData':
     built = language.code in languages_built
     if repo:
         completion, translators_data = get_completion(clones_dir, repo)
-        visitors_num = get_number_of_visitors(language.code, session) if built else 0
+        visitors_num = get_number_of_visitors(language.code, http) if built else 0
     else:
         completion = 0.0
         translators_data = TranslatorsData(0, False)
@@ -100,7 +99,8 @@ class LanguageProjectData:
 
 
 if __name__ == '__main__':
-    info(f'starting at {generation_time}')
+    logging.basicConfig(level=logging.INFO)
+    logging.info(f'starting at {generation_time}')
     template = Template(Path('template.html.jinja').read_text())
 
     output = template.render(

visitors.py

@@ -1,21 +1,23 @@
 import csv
 import io
-import urllib
+import urllib.parse
 import zipfile
 
-from requests import Session
+from urllib3 import PoolManager
 
 
-def get_number_of_visitors(language: str, requests: Session) -> int:
+def get_number_of_visitors(language: str, http: PoolManager) -> int:
     params = urllib.parse.urlencode(
         {'filters': f'[["contains","event:page",["/{language}/"]]]', 'period': 'all'}
     )
-    r = requests.get(
-        f'https://plausible.io/docs.python.org/export?{params}', timeout=20
-    )
+    r = http.request('GET', f'https://plausible.io/docs.python.org/export?{params}')
     with (
-        zipfile.ZipFile(io.BytesIO(r.content), 'r') as z,
+        zipfile.ZipFile(io.BytesIO(r.data), 'r') as z,
         z.open('visitors.csv') as csv_file,
     ):
         csv_reader = csv.DictReader(io.TextIOWrapper(csv_file))
         return sum(int(row['visitors']) for row in csv_reader)
+
+
+if __name__ == '__main__':
+    print(get_number_of_visitors('pl', PoolManager()))