|
9 | 9 | import time |
10 | 10 | from bisect import bisect_left |
11 | 11 | from collections import defaultdict |
12 | | -from datetime import datetime, timezone |
| 12 | +from datetime import datetime, timedelta, timezone # noqa: F401 |
13 | 13 | from importlib.metadata import metadata |
14 | 14 | from packaging.specifiers import SpecifierSet |
15 | 15 | from packaging.version import Version |
|
29 | 29 | from split_tox_gh_actions.split_tox_gh_actions import GROUPS |
30 | 30 |
|
31 | 31 |
|
| 32 | +# Set CUTOFF this to a datetime to ignore packages older than CUTOFF |
| 33 | +CUTOFF = None |
| 34 | +# CUTOFF = datetime.now(tz=timezone.utc) - timedelta(days=365 * 5) |
| 35 | + |
32 | 36 | TOX_FILE = Path(__file__).resolve().parent.parent.parent / "tox.ini" |
33 | 37 | ENV = Environment( |
34 | 38 | loader=FileSystemLoader(Path(__file__).resolve().parent), |
35 | 39 | trim_blocks=True, |
36 | 40 | lstrip_blocks=True, |
37 | 41 | ) |
38 | 42 |
|
| 43 | +PYPI_COOLDOWN = 0.15 # seconds to wait between requests to PyPI |
| 44 | + |
39 | 45 | PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json" |
40 | 46 | PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json" |
41 | 47 | CLASSIFIER_PREFIX = "Programming Language :: Python :: " |
|
68 | 74 | "beam", |
69 | 75 | "boto3", |
70 | 76 | "chalice", |
71 | | - "cohere", |
72 | | - "fastapi", |
73 | 77 | "gcp", |
74 | 78 | "httpx", |
75 | | - "huey", |
76 | | - "huggingface_hub", |
77 | 79 | "langchain", |
78 | 80 | "langchain_notiktoken", |
79 | 81 | "openai", |
|
88 | 90 | } |
89 | 91 |
|
90 | 92 |
|
91 | | -@functools.cache |
92 | | -def fetch_package(package: str) -> dict: |
93 | | - """Fetch package metadata from PyPI.""" |
94 | | - url = PYPI_PROJECT_URL.format(project=package) |
95 | | - pypi_data = requests.get(url) |
| 93 | +def fetch_url(url: str) -> Optional[dict]: |
| 94 | + for attempt in range(3): |
| 95 | + pypi_data = requests.get(url) |
| 96 | + |
| 97 | + if pypi_data.status_code == 200: |
| 98 | + return pypi_data.json() |
96 | 99 |
|
97 | | - if pypi_data.status_code != 200: |
98 | | - print(f"{package} not found") |
| 100 | + backoff = PYPI_COOLDOWN * 2**attempt |
| 101 | + print( |
| 102 | + f"{url} returned an error: {pypi_data.status_code}. Attempt {attempt + 1}/3. Waiting {backoff}s" |
| 103 | + ) |
| 104 | + time.sleep(backoff) |
99 | 105 |
|
100 | | - return pypi_data.json() |
| 106 | + return None |
101 | 107 |
|
102 | 108 |
|
103 | 109 | @functools.cache |
104 | | -def fetch_release(package: str, version: Version) -> dict: |
105 | | - url = PYPI_VERSION_URL.format(project=package, version=version) |
106 | | - pypi_data = requests.get(url) |
| 110 | +def fetch_package(package: str) -> Optional[dict]: |
| 111 | + """Fetch package metadata from PyPI.""" |
| 112 | + url = PYPI_PROJECT_URL.format(project=package) |
| 113 | + return fetch_url(url) |
107 | 114 |
|
108 | | - if pypi_data.status_code != 200: |
109 | | - print(f"{package} not found") |
110 | 115 |
|
111 | | - return pypi_data.json() |
| 116 | +@functools.cache |
| 117 | +def fetch_release(package: str, version: Version) -> Optional[dict]: |
| 118 | + """Fetch release metadata from PyPI.""" |
| 119 | + url = PYPI_VERSION_URL.format(project=package, version=version) |
| 120 | + return fetch_url(url) |
112 | 121 |
|
113 | 122 |
|
114 | 123 | def _prefilter_releases( |
@@ -153,9 +162,13 @@ def _prefilter_releases( |
153 | 162 | if meta["yanked"]: |
154 | 163 | continue |
155 | 164 |
|
156 | | - if older_than is not None: |
157 | | - if datetime.fromisoformat(meta["upload_time_iso_8601"]) > older_than: |
158 | | - continue |
| 165 | + uploaded = datetime.fromisoformat(meta["upload_time_iso_8601"]) |
| 166 | + |
| 167 | + if older_than is not None and uploaded > older_than: |
| 168 | + continue |
| 169 | + |
| 170 | + if CUTOFF is not None and uploaded < CUTOFF: |
| 171 | + continue |
159 | 172 |
|
160 | 173 | version = Version(release) |
161 | 174 |
|
@@ -221,16 +234,15 @@ def get_supported_releases( |
221 | 234 | integration, pypi_data["releases"], older_than |
222 | 235 | ) |
223 | 236 |
|
224 | | - # Determine Python support |
225 | | - expected_python_versions = TEST_SUITE_CONFIG[integration].get("python") |
226 | | - if expected_python_versions: |
227 | | - expected_python_versions = SpecifierSet(expected_python_versions) |
228 | | - else: |
229 | | - expected_python_versions = SpecifierSet(f">={MIN_PYTHON_VERSION}") |
230 | | - |
231 | 237 | def _supports_lowest(release: Version) -> bool: |
232 | | - time.sleep(0.1) # don't DoS PYPI |
233 | | - py_versions = determine_python_versions(fetch_release(package, release)) |
| 238 | + time.sleep(PYPI_COOLDOWN) # don't DoS PYPI |
| 239 | + |
| 240 | + pypi_data = fetch_release(package, release) |
| 241 | + if pypi_data is None: |
| 242 | + print("Failed to fetch necessary data from PyPI. Aborting.") |
| 243 | + sys.exit(1) |
| 244 | + |
| 245 | + py_versions = determine_python_versions(pypi_data) |
234 | 246 | target_python_versions = TEST_SUITE_CONFIG[integration].get("python") |
235 | 247 | if target_python_versions: |
236 | 248 | target_python_versions = SpecifierSet(target_python_versions) |
@@ -499,7 +511,11 @@ def _add_python_versions_to_release( |
499 | 511 | integration: str, package: str, release: Version |
500 | 512 | ) -> None: |
501 | 513 | release_pypi_data = fetch_release(package, release) |
502 | | - time.sleep(0.1) # give PYPI some breathing room |
| 514 | + if release_pypi_data is None: |
| 515 | + print("Failed to fetch necessary data from PyPI. Aborting.") |
| 516 | + sys.exit(1) |
| 517 | + |
| 518 | + time.sleep(PYPI_COOLDOWN) # give PYPI some breathing room |
503 | 519 |
|
504 | 520 | target_python_versions = TEST_SUITE_CONFIG[integration].get("python") |
505 | 521 | if target_python_versions: |
@@ -592,6 +608,9 @@ def main(fail_on_changes: bool = False) -> None: |
592 | 608 |
|
593 | 609 | # Fetch data for the main package |
594 | 610 | pypi_data = fetch_package(package) |
| 611 | + if pypi_data is None: |
| 612 | + print("Failed to fetch necessary data from PyPI. Aborting.") |
| 613 | + sys.exit(1) |
595 | 614 |
|
596 | 615 | # Get the list of all supported releases |
597 | 616 |
|
|
0 commit comments