wip

Author: sentrivana

scripts/populate_tox/populate_tox.py

@@ -1,23 +1,23 @@
 import configparser
 import functools
+import time
 from datetime import datetime, timedelta
 from pathlib import Path
 
 import requests
 
-from ..split_tox_gh_actions.split_tox_gh_actions import GROUPS
-
-print(GROUPS)
 
 # Only consider package versions going back this far
-CUTOFF = datetime.now() - timedelta(days=365 * 3)
+CUTOFF = datetime.now() - timedelta(days=365 * 5)
 LOWEST_SUPPORTED_PY_VERSION = "3.6"
 
 TOX_FILE = Path(__file__).resolve().parent.parent.parent / "tox.ini"
 
 PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json"
 PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json"
 
+CLASSIFIER_PREFIX = "Programming Language :: Python :: "
+
 EXCLUDE = {
     "common",
 }
@@ -27,7 +27,7 @@
 
 @functools.total_ordering
 class Version:
-    def __init__(self, version, metadata):
+    def __init__(self, version, metadata=None):
         self.raw = version
         self.metadata = metadata
 
@@ -36,10 +36,14 @@ def __init__(self, version, metadata):
         self.patch = None
         self.parsed = None
 
+        self.python_versions = []
+
         try:
             parsed = version.split(".")
-            if parsed[2].isnumeric():
+            if len(parsed) == 3 and parsed[2].isnumeric():
                 self.major, self.minor, self.patch = (int(p) for p in parsed)
+            elif len(parsed) == 2 and parsed[1].isnumeric():
+                self.major, self.minor = (int(p) for p in parsed)
         except Exception:
             # This will fail for e.g. prereleases, but we don't care about those
             # for now
@@ -97,50 +101,125 @@ def parse_tox():
             print(f"ERROR reading line {line}")
 
 
-def fetch_metadata(package):
+def fetch_package(package: str) -> dict:
+    """Fetch package metadata from PYPI."""
     url = PYPI_PROJECT_URL.format(project=package)
     pypi_data = requests.get(url)
 
     if pypi_data.status_code != 200:
         print(f"{package} not found")
 
-    import pprint
-
-    pprint.pprint(package)
-    pprint.pprint(pypi_data.json())
     return pypi_data.json()
 
 
-def parse_metadata(data):
-    package = data["info"]["name"]
+def get_releases(pypi_data: dict) -> list[Version]:
+    package = pypi_data["info"]["name"]
 
-    majors = {}
+    versions = []
+
+    for release, metadata in pypi_data["releases"].items():
+        if not metadata:
+            continue
 
-    for release, metadata in data["releases"].items():
         meta = metadata[0]
         if datetime.fromisoformat(meta["upload_time"]) < CUTOFF:
             continue
 
         version = Version(release, meta)
         if not version.valid:
-            print(f"Failed to parse version {release} of package {package}")
+            print(
+                f"Failed to parse version {release} of package {package}. Ignoring..."
+            )
             continue
 
-        if version.major not in majors:
-            # 0 -> [min 0.x version, max 0.x version]
-            majors[version.major] = [version, version]
-            continue
+        versions.append(version)
 
-        if version < majors[version.major][0]:
-            majors[version.major][0] = version
-        if version > majors[version.major][1]:
-            majors[version.major][1] = version
+    return sorted(versions)
+
+
+def pick_releases_to_test(releases: list[Version]) -> list[Version]:
+    indexes = [
+        0,  # oldest version younger than CUTOFF
+        len(releases) // 3,
+        len(releases) // 3 * 2,
+        -1,  # latest
+    ]
+    return [releases[i] for i in indexes]
+
+
+def fetch_release(package: str, version: Version) -> dict:
+    url = PYPI_VERSION_URL.format(project=package, version=version)
+    pypi_data = requests.get(url)
+
+    if pypi_data.status_code != 200:
+        print(f"{package} not found")
+
+    return pypi_data.json()
+
+
+def determine_python_versions(
+    package: str, version: Version, pypi_data: dict
+) -> list[str]:
+    try:
+        classifiers = pypi_data["info"]["classifiers"]
+    except (AttributeError, IndexError):
+        print(f"{package} {version} has no classifiers")
+        return []
+
+    python_versions = []
+    for classifier in classifiers:
+        if classifier.startswith(CLASSIFIER_PREFIX):
+            python_version = classifier[len(CLASSIFIER_PREFIX) :]
+            if "." in python_version:
+                # we don't care about stuff like
+                # Programming Language :: Python :: 3 :: Only
+                # Programming Language :: Python :: 3
+                # etc., we're only interested in specific versions like 3.13
+                python_versions.append(python_version)
+
+    python_versions = [
+        version
+        for version in python_versions
+        if Version(version) >= Version(LOWEST_SUPPORTED_PY_VERSION)
+    ]
+
+    return python_versions
+
+
+def write_tox_file(package, versions):
+    for version in versions:
+        print(
+            "{python_versions}-{package}-v{version}".format(
+                python_versions=",".join([f"py{v}" for v in version.python_versions]),
+                package=package,
+                version=version,
+            )
+        )
+
+    print()
+
+    for version in versions:
+        print(f"{package}-v{version}: {package}=={version.raw}")
+
+
+if __name__ == "__main__":
+    for package in ("celery", "django"):
+        pypi_data = fetch_package(package)
+        releases = get_releases(pypi_data)
+        test_releases = pick_releases_to_test(releases)
+        for release in test_releases:
+            release_pypi_data = fetch_release(package, release)
+            release.python_versions = determine_python_versions(
+                package, release, release_pypi_data
+            )
+            # XXX if no supported python versions -> delete
 
-        print(release, "not too old", meta["upload_time"])
+            print(release, " on ", release.python_versions)
+            time.sleep(0.1)
 
-    return majors
+        print(releases)
+        print(test_releases)
 
+        write_tox_file(package, test_releases)
 
-print(parse_tox())
-print(packages)
-print(parse_metadata(fetch_metadata("celery")))
+    print(parse_tox())