|
9 | 9 | import time |
10 | 10 | from bisect import bisect_left |
11 | 11 | from collections import defaultdict |
12 | | -from datetime import datetime, timezone |
| 12 | +from datetime import datetime, timedelta, timezone # noqa: F401 |
13 | 13 | from importlib.metadata import metadata |
14 | 14 | from packaging.specifiers import SpecifierSet |
15 | 15 | from packaging.version import Version |
|
29 | 29 | from split_tox_gh_actions.split_tox_gh_actions import GROUPS |
30 | 30 |
|
31 | 31 |
|
| 32 | +# Set CUTOFF this to a datetime to ignore packages older than CUTOFF |
| 33 | +CUTOFF = None |
| 34 | +# CUTOFF = datetime.now(tz=timezone.utc) - timedelta(days=365 * 5) |
| 35 | + |
32 | 36 | TOX_FILE = Path(__file__).resolve().parent.parent.parent / "tox.ini" |
33 | 37 | ENV = Environment( |
34 | 38 | loader=FileSystemLoader(Path(__file__).resolve().parent), |
35 | 39 | trim_blocks=True, |
36 | 40 | lstrip_blocks=True, |
37 | 41 | ) |
38 | 42 |
|
| 43 | +PYPI_COOLDOWN = 0.15 # seconds to wait between requests to PyPI |
| 44 | + |
39 | 45 | PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json" |
40 | 46 | PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json" |
41 | 47 | CLASSIFIER_PREFIX = "Programming Language :: Python :: " |
|
88 | 94 | } |
89 | 95 |
|
90 | 96 |
|
91 | | -@functools.cache |
92 | | -def fetch_package(package: str) -> dict: |
93 | | - """Fetch package metadata from PyPI.""" |
94 | | - url = PYPI_PROJECT_URL.format(project=package) |
95 | | - pypi_data = requests.get(url) |
| 97 | +def fetch_url(url: str) -> Optional[dict]: |
| 98 | + for attempt in range(3): |
| 99 | + pypi_data = requests.get(url) |
96 | 100 |
|
97 | | - if pypi_data.status_code != 200: |
98 | | - print(f"{package} not found") |
| 101 | + if pypi_data.status_code == 200: |
| 102 | + return pypi_data.json() |
99 | 103 |
|
100 | | - return pypi_data.json() |
| 104 | + backoff = PYPI_COOLDOWN * 2**attempt |
| 105 | + print( |
| 106 | + f"{url} returned an error: {pypi_data.status_code}. Attempt {attempt + 1}/3. Waiting {backoff}s" |
| 107 | + ) |
| 108 | + time.sleep(backoff) |
| 109 | + |
| 110 | + return None |
101 | 111 |
|
102 | 112 |
|
103 | 113 | @functools.cache |
104 | | -def fetch_release(package: str, version: Version) -> dict: |
105 | | - url = PYPI_VERSION_URL.format(project=package, version=version) |
106 | | - pypi_data = requests.get(url) |
| 114 | +def fetch_package(package: str) -> Optional[dict]: |
| 115 | + """Fetch package metadata from PyPI.""" |
| 116 | + url = PYPI_PROJECT_URL.format(project=package) |
| 117 | + return fetch_url(url) |
107 | 118 |
|
108 | | - if pypi_data.status_code != 200: |
109 | | - print(f"{package} not found") |
110 | 119 |
|
111 | | - return pypi_data.json() |
| 120 | +@functools.cache |
| 121 | +def fetch_release(package: str, version: Version) -> Optional[dict]: |
| 122 | + """Fetch release metadata from PyPI.""" |
| 123 | + url = PYPI_VERSION_URL.format(project=package, version=version) |
| 124 | + return fetch_url(url) |
112 | 125 |
|
113 | 126 |
|
114 | 127 | def _prefilter_releases( |
@@ -153,9 +166,13 @@ def _prefilter_releases( |
153 | 166 | if meta["yanked"]: |
154 | 167 | continue |
155 | 168 |
|
156 | | - if older_than is not None: |
157 | | - if datetime.fromisoformat(meta["upload_time_iso_8601"]) > older_than: |
158 | | - continue |
| 169 | + uploaded = datetime.fromisoformat(meta["upload_time_iso_8601"]) |
| 170 | + |
| 171 | + if older_than is not None and uploaded > older_than: |
| 172 | + continue |
| 173 | + |
| 174 | + if CUTOFF is not None and uploaded < CUTOFF: |
| 175 | + continue |
159 | 176 |
|
160 | 177 | version = Version(release) |
161 | 178 |
|
@@ -229,8 +246,14 @@ def get_supported_releases( |
229 | 246 | expected_python_versions = SpecifierSet(f">={MIN_PYTHON_VERSION}") |
230 | 247 |
|
231 | 248 | def _supports_lowest(release: Version) -> bool: |
232 | | - time.sleep(0.1) # don't DoS PYPI |
233 | | - py_versions = determine_python_versions(fetch_release(package, release)) |
| 249 | + time.sleep(PYPI_COOLDOWN) # don't DoS PYPI |
| 250 | + |
| 251 | + pypi_data = fetch_release(package, release) |
| 252 | + if pypi_data is None: |
| 253 | + print("Failed to fetch necessary data from PyPI. Aborting.") |
| 254 | + sys.exit(1) |
| 255 | + |
| 256 | + py_versions = determine_python_versions(pypi_data) |
234 | 257 | target_python_versions = TEST_SUITE_CONFIG[integration].get("python") |
235 | 258 | if target_python_versions: |
236 | 259 | target_python_versions = SpecifierSet(target_python_versions) |
@@ -499,7 +522,11 @@ def _add_python_versions_to_release( |
499 | 522 | integration: str, package: str, release: Version |
500 | 523 | ) -> None: |
501 | 524 | release_pypi_data = fetch_release(package, release) |
502 | | - time.sleep(0.1) # give PYPI some breathing room |
| 525 | + if release_pypi_data is None: |
| 526 | + print("Failed to fetch necessary data from PyPI. Aborting.") |
| 527 | + sys.exit(1) |
| 528 | + |
| 529 | + time.sleep(PYPI_COOLDOWN) # give PYPI some breathing room |
503 | 530 |
|
504 | 531 | target_python_versions = TEST_SUITE_CONFIG[integration].get("python") |
505 | 532 | if target_python_versions: |
@@ -592,6 +619,9 @@ def main(fail_on_changes: bool = False) -> None: |
592 | 619 |
|
593 | 620 | # Fetch data for the main package |
594 | 621 | pypi_data = fetch_package(package) |
| 622 | + if pypi_data is None: |
| 623 | + print("Failed to fetch necessary data from PyPI. Aborting.") |
| 624 | + sys.exit(1) |
595 | 625 |
|
596 | 626 | # Get the list of all supported releases |
597 | 627 |
|
|
0 commit comments