From 4925b2078b46985b2581c58ef237fab4efb8361b Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 13:27:39 +0100 Subject: [PATCH 1/8] tests: Add fail_on_changes to toxgen --- scripts/populate_tox/populate_tox.py | 110 +++++++++++++++++++++++++-- scripts/populate_tox/tox.jinja | 2 + tox.ini | 72 ++++++++---------- 3 files changed, 136 insertions(+), 48 deletions(-) diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index 4bfce80ce7..1583d90a20 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -3,11 +3,13 @@ """ import functools +import hashlib import os import sys import time from bisect import bisect_left from collections import defaultdict +from datetime import datetime, timezone from importlib.metadata import metadata from packaging.specifiers import SpecifierSet from packaging.version import Version @@ -106,7 +108,9 @@ def fetch_release(package: str, version: Version) -> dict: return pypi_data.json() -def _prefilter_releases(integration: str, releases: dict[str, dict]) -> list[Version]: +def _prefilter_releases( + integration: str, releases: dict[str, dict], older_than: Optional[datetime] = None +) -> list[Version]: """ Filter `releases`, removing releases that are for sure unsupported. @@ -135,6 +139,10 @@ def _prefilter_releases(integration: str, releases: dict[str, dict]) -> list[Ver if meta["yanked"]: continue + if older_than is not None: + if datetime.fromisoformat(meta["upload_time_iso_8601"]) > older_than: + continue + version = Version(release) if min_supported and version < min_supported: @@ -160,19 +168,24 @@ def _prefilter_releases(integration: str, releases: dict[str, dict]) -> list[Ver return sorted(filtered_releases) -def get_supported_releases(integration: str, pypi_data: dict) -> list[Version]: +def get_supported_releases( + integration: str, pypi_data: dict, older_than: Optional[datetime] = None +) -> list[Version]: """ Get a list of releases that are currently supported by the SDK. This takes into account a handful of parameters (Python support, the lowest version we've defined for the framework, the date of the release). + + If an `older_than` timestamp is provided, no release newer than that will be + considered. """ package = pypi_data["info"]["name"] # Get a consolidated list without taking into account Python support yet # (because that might require an additional API call for some # of the releases) - releases = _prefilter_releases(integration, pypi_data["releases"]) + releases = _prefilter_releases(integration, pypi_data["releases"], older_than) # Determine Python support expected_python_versions = TEST_SUITE_CONFIG[integration].get("python") @@ -381,7 +394,9 @@ def _render_dependencies(integration: str, releases: list[Version]) -> list[str] return rendered -def write_tox_file(packages: dict) -> None: +def write_tox_file( + packages: dict, update_timestamp: bool, last_updated: datetime +) -> None: template = ENV.get_template("tox.jinja") context = {"groups": {}} @@ -400,6 +415,11 @@ def write_tox_file(packages: dict) -> None: } ) + if update_timestamp: + context["updated"] = datetime.now(tz=timezone.utc).isoformat() + else: + context["updated"] = last_updated.isoformat() + rendered = template.render(context) with open(TOX_FILE, "w") as file: @@ -453,7 +473,52 @@ def _add_python_versions_to_release( release.rendered_python_versions = _render_python_versions(release.python_versions) -def main() -> None: +def get_file_hash() -> str: + """Calculate a hash of the tox.ini file.""" + hasher = hashlib.md5() + + with open(TOX_FILE, "rb") as f: + buf = f.read() + hasher.update(buf) + + return hasher.hexdigest() + + +def get_last_updated() -> Optional[datetime]: + timestamp = None + + with open(TOX_FILE, "r") as f: + for line in f: + if line.startswith("# Last generated:"): + timestamp = datetime.fromisoformat(line.strip().split[-1]) + break + + if timestamp is None: + print( + "Failed to find out when tox.ini was last generated; the timestamp seems to be missing from the file." + ) + + return timestamp + + +def main(fail_on_changes: bool = False) -> None: + """ + Generate tox.ini from the tox.jinja template. + + The script has two modes of operation: + - check mode (if `fail_on_changes` is True) + - normal mode (if `fail_on_changes` is False) + + Check mode is run on every PR to make sure that `tox.ini`, `tox.jinja` and + this script don't go out of sync because of manual changes in one place but + not the other. + + Normal mode is meant to be run as a cron job, regenerating tox.ini and + proposing the changes via a PR. + """ + mode = "check" if fail_on_changes else "normal" + print(f"Running in {mode} mode.") + global MIN_PYTHON_VERSION, MAX_PYTHON_VERSION sdk_python_versions = _parse_python_versions_from_classifiers( metadata("sentry-sdk").get_all("Classifier") @@ -464,6 +529,12 @@ def main() -> None: f"The SDK supports Python versions {MIN_PYTHON_VERSION} - {MAX_PYTHON_VERSION}." ) + # If this script is run in check mode (fail_on_changes is True), we need to + # make the script ignore any new releases after the `last_updated` timestamp + # so that we don't fail CI on a PR just because a new package version was + # released, leading to unrelated changes in tox.ini. + last_updated = get_last_updated() + packages = defaultdict(list) for group, integrations in GROUPS.items(): @@ -480,7 +551,9 @@ def main() -> None: pypi_data = fetch_package(package) # Get the list of all supported releases - releases = get_supported_releases(integration, pypi_data) + # If in check mode, ignore releases newer than `last_updated` + older_than = last_updated if fail_on_changes else None + releases = get_supported_releases(integration, pypi_data, older_than) if not releases: print(" Found no supported releases.") continue @@ -510,8 +583,29 @@ def main() -> None: } ) - write_tox_file(packages) + if fail_on_changes: + old_file_hash = get_file_hash() + + write_tox_file( + packages, update_timestamp=not fail_on_changes, last_updated=last_updated + ) + + if fail_on_changes: + new_file_hash = get_file_hash() + if old_file_hash != new_file_hash: + raise RuntimeError( + "The yaml configuration files have changed. This means that either `tox.ini` " + "or one of the constants in `split_tox_gh_actions.py` has changed " + "but the changes have not been propagated to the GitHub actions config files. " + "Please run `python scripts/split_tox_gh_actions/split_tox_gh_actions.py` " + "locally and commit the changes of the yaml configuration files to continue. " + ) + + print( + "Done generating tox.ini. Make sure to also update the CI YAML files to reflect the new test targets." + ) if __name__ == "__main__": - main() + fail_on_changes = len(sys.argv) == 2 and sys.argv[1] == "--fail-on-changes" + main(fail_on_changes) diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index 15119b4768..81ab17c919 100644 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -9,6 +9,8 @@ # or in the script (if you want to change the auto-generated part). # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". +# +# Last generated: {{ updated }} [tox] requires = diff --git a/tox.ini b/tox.ini index 9ce3d40a21..764dac930d 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,8 @@ # or in the script (if you want to change the auto-generated part). # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". +# +# Last generated: 2025-02-18T12:26:42.990713+00:00 [tox] requires = @@ -176,17 +178,16 @@ envlist = # ~~~ DBs ~~~ {py3.7,py3.11,py3.12}-clickhouse_driver-v0.2.9 - {py3.6}-pymongo-v3.5.1 - {py3.6,py3.10,py3.11}-pymongo-v3.13.0 - {py3.6,py3.9,py3.10}-pymongo-v4.0.2 + {py3.7}-pymongo-v3.7.2 + {py3.7,py3.10,py3.11}-pymongo-v3.13.0 + {py3.7,py3.9,py3.10}-pymongo-v4.0.2 {py3.9,py3.12,py3.13}-pymongo-v4.11.1 - {py3.6}-redis_py_cluster_legacy-v1.3.6 - {py3.6,py3.7}-redis_py_cluster_legacy-v2.0.0 - {py3.6,py3.7,py3.8}-redis_py_cluster_legacy-v2.1.3 + {py3.7}-redis_py_cluster_legacy-v2.0.0 + {py3.7,py3.8}-redis_py_cluster_legacy-v2.1.3 - {py3.6,py3.7}-sqlalchemy-v1.3.9 - {py3.6,py3.11,py3.12}-sqlalchemy-v1.4.54 + {py3.7}-sqlalchemy-v1.3.9 + {py3.7,py3.11,py3.12}-sqlalchemy-v1.4.54 {py3.7,py3.10,py3.11}-sqlalchemy-v2.0.9 {py3.7,py3.12,py3.13}-sqlalchemy-v2.0.38 @@ -211,10 +212,10 @@ envlist = {py3.8,py3.11,py3.12}-ariadne-v0.24.0 {py3.8,py3.11,py3.12}-ariadne-v0.25.2 - {py3.6,py3.9,py3.10}-gql-v3.4.1 + {py3.7,py3.9,py3.10}-gql-v3.4.1 {py3.7,py3.11,py3.12}-gql-v3.5.0 - {py3.6,py3.9,py3.10}-graphene-v3.3 + {py3.7,py3.9,py3.10}-graphene-v3.3 {py3.8,py3.12,py3.13}-graphene-v3.4.3 {py3.8,py3.10,py3.11}-strawberry-v0.209.8 @@ -231,12 +232,12 @@ envlist = # ~~~ Tasks ~~~ - {py3.6,py3.7,py3.8}-celery-v4.4.7 - {py3.6,py3.7,py3.8}-celery-v5.0.5 + {py3.7,py3.8}-celery-v4.4.7 + {py3.7,py3.8}-celery-v5.0.5 {py3.8,py3.11,py3.12}-celery-v5.4.0 - {py3.6,py3.7}-dramatiq-v1.9.0 - {py3.6,py3.8,py3.9}-dramatiq-v1.12.3 + {py3.7}-dramatiq-v1.9.0 + {py3.7,py3.8,py3.9}-dramatiq-v1.12.3 {py3.7,py3.10,py3.11}-dramatiq-v1.15.0 {py3.8,py3.12,py3.13}-dramatiq-v1.17.1 @@ -247,50 +248,47 @@ envlist = # ~~~ Web 1 ~~~ - {py3.6,py3.7,py3.8}-flask-v1.1.4 + {py3.7,py3.8}-flask-v1.1.4 {py3.8,py3.12,py3.13}-flask-v2.3.3 {py3.8,py3.12,py3.13}-flask-v3.0.3 {py3.9,py3.12,py3.13}-flask-v3.1.0 - {py3.6,py3.9,py3.10}-starlette-v0.16.0 + {py3.7,py3.9,py3.10}-starlette-v0.16.0 {py3.7,py3.10,py3.11}-starlette-v0.26.1 {py3.8,py3.11,py3.12}-starlette-v0.36.3 {py3.9,py3.12,py3.13}-starlette-v0.45.3 # ~~~ Web 2 ~~~ - {py3.6,py3.7}-bottle-v0.12.25 - {py3.6,py3.8,py3.9}-bottle-v0.13.2 + {py3.7}-bottle-v0.12.25 + {py3.7,py3.8,py3.9}-bottle-v0.13.2 - {py3.6}-falcon-v1.4.1 - {py3.6,py3.7}-falcon-v2.0.0 - {py3.6,py3.11,py3.12}-falcon-v3.1.3 + {py3.7}-falcon-v2.0.0 + {py3.7,py3.11,py3.12}-falcon-v3.1.3 {py3.8,py3.11,py3.12}-falcon-v4.0.2 - {py3.6}-pyramid-v1.8.6 - {py3.6,py3.8,py3.9}-pyramid-v1.10.8 - {py3.6,py3.10,py3.11}-pyramid-v2.0.2 + {py3.7,py3.8,py3.9}-pyramid-v1.10.8 + {py3.7,py3.10,py3.11}-pyramid-v2.0.2 {py3.8,py3.10,py3.11}-starlite-v1.48.1 {py3.8,py3.10,py3.11}-starlite-v1.49.0 {py3.8,py3.10,py3.11}-starlite-v1.50.2 {py3.8,py3.10,py3.11}-starlite-v1.51.16 - {py3.6,py3.7,py3.8}-tornado-v6.0.4 - {py3.6,py3.8,py3.9}-tornado-v6.1 + {py3.7,py3.8}-tornado-v6.0.4 + {py3.7,py3.8,py3.9}-tornado-v6.1 {py3.7,py3.9,py3.10}-tornado-v6.2 {py3.8,py3.10,py3.11}-tornado-v6.4.2 # ~~~ Misc ~~~ - {py3.6,py3.12,py3.13}-loguru-v0.7.3 + {py3.7,py3.12,py3.13}-loguru-v0.7.3 - {py3.6}-trytond-v4.6.9 - {py3.6}-trytond-v4.8.18 - {py3.6,py3.7,py3.8}-trytond-v5.8.16 + {py3.7}-trytond-v5.0.9 + {py3.7,py3.8}-trytond-v5.8.16 {py3.8,py3.10,py3.11}-trytond-v6.8.17 {py3.8,py3.11,py3.12}-trytond-v7.0.9 - {py3.8,py3.11,py3.12}-trytond-v7.4.5 + {py3.8,py3.11,py3.12}-trytond-v7.4.6 {py3.7,py3.11,py3.12}-typer-v0.15.1 @@ -555,13 +553,12 @@ deps = # ~~~ DBs ~~~ clickhouse_driver-v0.2.9: clickhouse-driver==0.2.9 - pymongo-v3.5.1: pymongo==3.5.1 + pymongo-v3.7.2: pymongo==3.7.2 pymongo-v3.13.0: pymongo==3.13.0 pymongo-v4.0.2: pymongo==4.0.2 pymongo-v4.11.1: pymongo==4.11.1 pymongo: mockupdb - redis_py_cluster_legacy-v1.3.6: redis-py-cluster==1.3.6 redis_py_cluster_legacy-v2.0.0: redis-py-cluster==2.0.0 redis_py_cluster_legacy-v2.1.3: redis-py-cluster==2.1.3 @@ -674,12 +671,10 @@ deps = bottle-v0.13.2: bottle==0.13.2 bottle: werkzeug<2.1.0 - falcon-v1.4.1: falcon==1.4.1 falcon-v2.0.0: falcon==2.0.0 falcon-v3.1.3: falcon==3.1.3 falcon-v4.0.2: falcon==4.0.2 - pyramid-v1.8.6: pyramid==1.8.6 pyramid-v1.10.8: pyramid==1.10.8 pyramid-v2.0.2: pyramid==2.0.2 pyramid: werkzeug<2.1.0 @@ -709,15 +704,12 @@ deps = # ~~~ Misc ~~~ loguru-v0.7.3: loguru==0.7.3 - trytond-v4.6.9: trytond==4.6.9 - trytond-v4.8.18: trytond==4.8.18 + trytond-v5.0.9: trytond==5.0.9 trytond-v5.8.16: trytond==5.8.16 trytond-v6.8.17: trytond==6.8.17 trytond-v7.0.9: trytond==7.0.9 - trytond-v7.4.5: trytond==7.4.5 + trytond-v7.4.6: trytond==7.4.6 trytond: werkzeug - trytond-v4.6.9: werkzeug<1.0 - trytond-v4.8.18: werkzeug<1.0 typer-v0.15.1: typer==0.15.1 From 3dd520cb47fd2ef33a7009aaa7f1803e4bbbea98 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 13:42:48 +0100 Subject: [PATCH 2/8] add to ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8931e229e..55b5300378 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,8 @@ jobs: python-version: 3.12 - run: | + pip install -r scripts/populate_tox/requirements.txt + python scripts/populate_tox/populate_tox.py --fail-on-changes pip install -r scripts/split_tox_gh_actions/requirements.txt python scripts/split_tox_gh_actions/split_tox_gh_actions.py --fail-on-changes From 954bb1bc309589f9ba413dd1666f8524001a5fa3 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 13:44:47 +0100 Subject: [PATCH 3/8] rerun --- scripts/populate_tox/populate_tox.py | 2 +- tox.ini | 68 ++++++++++++++++------------ 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index 1583d90a20..59d34cf33e 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -490,7 +490,7 @@ def get_last_updated() -> Optional[datetime]: with open(TOX_FILE, "r") as f: for line in f: if line.startswith("# Last generated:"): - timestamp = datetime.fromisoformat(line.strip().split[-1]) + timestamp = datetime.fromisoformat(line.strip().split()[-1]) break if timestamp is None: diff --git a/tox.ini b/tox.ini index 764dac930d..8b287b3077 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-02-18T12:26:42.990713+00:00 +# Last generated: 2025-02-18T12:44:30.669986+00:00 [tox] requires = @@ -178,16 +178,17 @@ envlist = # ~~~ DBs ~~~ {py3.7,py3.11,py3.12}-clickhouse_driver-v0.2.9 - {py3.7}-pymongo-v3.7.2 - {py3.7,py3.10,py3.11}-pymongo-v3.13.0 - {py3.7,py3.9,py3.10}-pymongo-v4.0.2 + {py3.6}-pymongo-v3.5.1 + {py3.6,py3.10,py3.11}-pymongo-v3.13.0 + {py3.6,py3.9,py3.10}-pymongo-v4.0.2 {py3.9,py3.12,py3.13}-pymongo-v4.11.1 - {py3.7}-redis_py_cluster_legacy-v2.0.0 - {py3.7,py3.8}-redis_py_cluster_legacy-v2.1.3 + {py3.6}-redis_py_cluster_legacy-v1.3.6 + {py3.6,py3.7}-redis_py_cluster_legacy-v2.0.0 + {py3.6,py3.7,py3.8}-redis_py_cluster_legacy-v2.1.3 - {py3.7}-sqlalchemy-v1.3.9 - {py3.7,py3.11,py3.12}-sqlalchemy-v1.4.54 + {py3.6,py3.7}-sqlalchemy-v1.3.9 + {py3.6,py3.11,py3.12}-sqlalchemy-v1.4.54 {py3.7,py3.10,py3.11}-sqlalchemy-v2.0.9 {py3.7,py3.12,py3.13}-sqlalchemy-v2.0.38 @@ -212,10 +213,10 @@ envlist = {py3.8,py3.11,py3.12}-ariadne-v0.24.0 {py3.8,py3.11,py3.12}-ariadne-v0.25.2 - {py3.7,py3.9,py3.10}-gql-v3.4.1 + {py3.6,py3.9,py3.10}-gql-v3.4.1 {py3.7,py3.11,py3.12}-gql-v3.5.0 - {py3.7,py3.9,py3.10}-graphene-v3.3 + {py3.6,py3.9,py3.10}-graphene-v3.3 {py3.8,py3.12,py3.13}-graphene-v3.4.3 {py3.8,py3.10,py3.11}-strawberry-v0.209.8 @@ -232,12 +233,12 @@ envlist = # ~~~ Tasks ~~~ - {py3.7,py3.8}-celery-v4.4.7 - {py3.7,py3.8}-celery-v5.0.5 + {py3.6,py3.7,py3.8}-celery-v4.4.7 + {py3.6,py3.7,py3.8}-celery-v5.0.5 {py3.8,py3.11,py3.12}-celery-v5.4.0 - {py3.7}-dramatiq-v1.9.0 - {py3.7,py3.8,py3.9}-dramatiq-v1.12.3 + {py3.6,py3.7}-dramatiq-v1.9.0 + {py3.6,py3.8,py3.9}-dramatiq-v1.12.3 {py3.7,py3.10,py3.11}-dramatiq-v1.15.0 {py3.8,py3.12,py3.13}-dramatiq-v1.17.1 @@ -248,44 +249,47 @@ envlist = # ~~~ Web 1 ~~~ - {py3.7,py3.8}-flask-v1.1.4 + {py3.6,py3.7,py3.8}-flask-v1.1.4 {py3.8,py3.12,py3.13}-flask-v2.3.3 {py3.8,py3.12,py3.13}-flask-v3.0.3 {py3.9,py3.12,py3.13}-flask-v3.1.0 - {py3.7,py3.9,py3.10}-starlette-v0.16.0 + {py3.6,py3.9,py3.10}-starlette-v0.16.0 {py3.7,py3.10,py3.11}-starlette-v0.26.1 {py3.8,py3.11,py3.12}-starlette-v0.36.3 {py3.9,py3.12,py3.13}-starlette-v0.45.3 # ~~~ Web 2 ~~~ - {py3.7}-bottle-v0.12.25 - {py3.7,py3.8,py3.9}-bottle-v0.13.2 + {py3.6,py3.7}-bottle-v0.12.25 + {py3.6,py3.8,py3.9}-bottle-v0.13.2 - {py3.7}-falcon-v2.0.0 - {py3.7,py3.11,py3.12}-falcon-v3.1.3 + {py3.6}-falcon-v1.4.1 + {py3.6,py3.7}-falcon-v2.0.0 + {py3.6,py3.11,py3.12}-falcon-v3.1.3 {py3.8,py3.11,py3.12}-falcon-v4.0.2 - {py3.7,py3.8,py3.9}-pyramid-v1.10.8 - {py3.7,py3.10,py3.11}-pyramid-v2.0.2 + {py3.6}-pyramid-v1.8.6 + {py3.6,py3.8,py3.9}-pyramid-v1.10.8 + {py3.6,py3.10,py3.11}-pyramid-v2.0.2 {py3.8,py3.10,py3.11}-starlite-v1.48.1 {py3.8,py3.10,py3.11}-starlite-v1.49.0 {py3.8,py3.10,py3.11}-starlite-v1.50.2 {py3.8,py3.10,py3.11}-starlite-v1.51.16 - {py3.7,py3.8}-tornado-v6.0.4 - {py3.7,py3.8,py3.9}-tornado-v6.1 + {py3.6,py3.7,py3.8}-tornado-v6.0.4 + {py3.6,py3.8,py3.9}-tornado-v6.1 {py3.7,py3.9,py3.10}-tornado-v6.2 {py3.8,py3.10,py3.11}-tornado-v6.4.2 # ~~~ Misc ~~~ - {py3.7,py3.12,py3.13}-loguru-v0.7.3 + {py3.6,py3.12,py3.13}-loguru-v0.7.3 - {py3.7}-trytond-v5.0.9 - {py3.7,py3.8}-trytond-v5.8.16 + {py3.6}-trytond-v4.6.9 + {py3.6}-trytond-v4.8.18 + {py3.6,py3.7,py3.8}-trytond-v5.8.16 {py3.8,py3.10,py3.11}-trytond-v6.8.17 {py3.8,py3.11,py3.12}-trytond-v7.0.9 {py3.8,py3.11,py3.12}-trytond-v7.4.6 @@ -553,12 +557,13 @@ deps = # ~~~ DBs ~~~ clickhouse_driver-v0.2.9: clickhouse-driver==0.2.9 - pymongo-v3.7.2: pymongo==3.7.2 + pymongo-v3.5.1: pymongo==3.5.1 pymongo-v3.13.0: pymongo==3.13.0 pymongo-v4.0.2: pymongo==4.0.2 pymongo-v4.11.1: pymongo==4.11.1 pymongo: mockupdb + redis_py_cluster_legacy-v1.3.6: redis-py-cluster==1.3.6 redis_py_cluster_legacy-v2.0.0: redis-py-cluster==2.0.0 redis_py_cluster_legacy-v2.1.3: redis-py-cluster==2.1.3 @@ -671,10 +676,12 @@ deps = bottle-v0.13.2: bottle==0.13.2 bottle: werkzeug<2.1.0 + falcon-v1.4.1: falcon==1.4.1 falcon-v2.0.0: falcon==2.0.0 falcon-v3.1.3: falcon==3.1.3 falcon-v4.0.2: falcon==4.0.2 + pyramid-v1.8.6: pyramid==1.8.6 pyramid-v1.10.8: pyramid==1.10.8 pyramid-v2.0.2: pyramid==2.0.2 pyramid: werkzeug<2.1.0 @@ -704,12 +711,15 @@ deps = # ~~~ Misc ~~~ loguru-v0.7.3: loguru==0.7.3 - trytond-v5.0.9: trytond==5.0.9 + trytond-v4.6.9: trytond==4.6.9 + trytond-v4.8.18: trytond==4.8.18 trytond-v5.8.16: trytond==5.8.16 trytond-v6.8.17: trytond==6.8.17 trytond-v7.0.9: trytond==7.0.9 trytond-v7.4.6: trytond==7.4.6 trytond: werkzeug + trytond-v4.6.9: werkzeug<1.0 + trytond-v4.8.18: werkzeug<1.0 typer-v0.15.1: typer==0.15.1 From bbe49d491d894eb9cee163aeb78d662adbcd21ec Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 14:01:24 +0100 Subject: [PATCH 4/8] . --- scripts/populate_tox/populate_tox.py | 30 +++++++++++++++++++--------- tox.ini | 2 +- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index 59d34cf33e..4a61df8374 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -14,6 +14,7 @@ from packaging.specifiers import SpecifierSet from packaging.version import Version from pathlib import Path +from textwrap import dedent from typing import Optional, Union # Adding the scripts directory to PATH. This is necessary in order to be able @@ -594,16 +595,27 @@ def main(fail_on_changes: bool = False) -> None: new_file_hash = get_file_hash() if old_file_hash != new_file_hash: raise RuntimeError( - "The yaml configuration files have changed. This means that either `tox.ini` " - "or one of the constants in `split_tox_gh_actions.py` has changed " - "but the changes have not been propagated to the GitHub actions config files. " - "Please run `python scripts/split_tox_gh_actions/split_tox_gh_actions.py` " - "locally and commit the changes of the yaml configuration files to continue. " + dedent( + """ + Detected an unexpected change in `tox.ini` that is not reflected + in `tox.jinja` and/or `populate_tox.py`. + + Please make sure to not make manual changes to `tox.ini`. The file + is generated from a template in `scripts/populate_tox/tox.jinja` + by the `scripts/populate_tox/populate_tox.py` script. Any changes + should be made to the template or the script directly and the + resulting `tox.ini` file should be generated with: + + pip install -r scripts/populate_tox/requirements.txt + python scripts/populate_tox/populate_tox.py + """ + ) ) - - print( - "Done generating tox.ini. Make sure to also update the CI YAML files to reflect the new test targets." - ) + print("Done checking tox.ini. Looking good!") + else: + print( + "Done generating tox.ini. Make sure to also update the CI YAML files to reflect the new test targets." + ) if __name__ == "__main__": diff --git a/tox.ini b/tox.ini index 8b287b3077..0e41500fe1 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-02-18T12:44:30.669986+00:00 +# Last generated: 2025-02-18T12:57:32.874168+00:00 [tox] requires = From 727619353c0307da634722d3e2ffc30b3ca6de9e Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 15:36:15 +0100 Subject: [PATCH 5/8] install sdk --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55b5300378..e4c76c7245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: python-version: 3.12 - run: | + pip install -e . pip install -r scripts/populate_tox/requirements.txt python scripts/populate_tox/populate_tox.py --fail-on-changes pip install -r scripts/split_tox_gh_actions/requirements.txt From 148ea3386c9d95359431150c6fc369048b7ca3ac Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 16:06:50 +0100 Subject: [PATCH 6/8] more prints --- scripts/populate_tox/populate_tox.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index 4a61df8374..f1d8dd8dc0 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -507,18 +507,25 @@ def main(fail_on_changes: bool = False) -> None: Generate tox.ini from the tox.jinja template. The script has two modes of operation: - - check mode (if `fail_on_changes` is True) + - fail on changes mode (if `fail_on_changes` is True) - normal mode (if `fail_on_changes` is False) - Check mode is run on every PR to make sure that `tox.ini`, `tox.jinja` and - this script don't go out of sync because of manual changes in one place but - not the other. + Fail on changes mode is run on every PR to make sure that `tox.ini`, + `tox.jinja` and this script don't go out of sync because of manual changes + in one place but not the other. Normal mode is meant to be run as a cron job, regenerating tox.ini and proposing the changes via a PR. """ - mode = "check" if fail_on_changes else "normal" - print(f"Running in {mode} mode.") + print(f"Running in {'fail_on_changes' if fail_on_changes else 'normal'} mode.") + last_updated = get_last_updated() + if fail_on_changes: + # We need to make the script ignore any new releases after the `last_updated` + # timestamp so that we don't fail CI on a PR just because a new package + # version was released, leading to unrelated changes in tox.ini. + print( + f"Since we're in fail_on_changes mode, we're only considering releases before the last tox.ini update at {last_updated.isoformat()}." + ) global MIN_PYTHON_VERSION, MAX_PYTHON_VERSION sdk_python_versions = _parse_python_versions_from_classifiers( @@ -530,12 +537,6 @@ def main(fail_on_changes: bool = False) -> None: f"The SDK supports Python versions {MIN_PYTHON_VERSION} - {MAX_PYTHON_VERSION}." ) - # If this script is run in check mode (fail_on_changes is True), we need to - # make the script ignore any new releases after the `last_updated` timestamp - # so that we don't fail CI on a PR just because a new package version was - # released, leading to unrelated changes in tox.ini. - last_updated = get_last_updated() - packages = defaultdict(list) for group, integrations in GROUPS.items(): From 48a11adc0f9f0b1e23169779d1afee3fa3bc8f02 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Tue, 18 Feb 2025 16:10:22 +0100 Subject: [PATCH 7/8] nicer name for step --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4c76c7245..03ed8de742 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,8 @@ jobs: with: python-version: 3.12 - - run: | + - name: Detect unexpected changes to tox.ini or CI + run: | pip install -e . pip install -r scripts/populate_tox/requirements.txt python scripts/populate_tox/populate_tox.py --fail-on-changes From 0524b69b4f8dc62bd2300c5b37c230cddb06b66a Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 19 Feb 2025 10:00:52 +0100 Subject: [PATCH 8/8] wording --- scripts/populate_tox/populate_tox.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index f1d8dd8dc0..5906eee5b4 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -598,15 +598,19 @@ def main(fail_on_changes: bool = False) -> None: raise RuntimeError( dedent( """ - Detected an unexpected change in `tox.ini` that is not reflected - in `tox.jinja` and/or `populate_tox.py`. - - Please make sure to not make manual changes to `tox.ini`. The file - is generated from a template in `scripts/populate_tox/tox.jinja` - by the `scripts/populate_tox/populate_tox.py` script. Any changes - should be made to the template or the script directly and the - resulting `tox.ini` file should be generated with: - + Detected that `tox.ini` is out of sync with + `scripts/populate_tox/tox.jinja` and/or + `scripts/populate_tox/populate_tox.py`. This might either mean + that `tox.ini` was changed manually, or the `tox.jinja` + template and/or the `populate_tox.py` script were changed without + regenerating `tox.ini`. + + Please don't make manual changes to `tox.ini`. Instead, make the + changes to the `tox.jinja` template and/or the `populate_tox.py` + script (as applicable) and regenerate the `tox.ini` file with: + + python -m venv toxgen.env + . toxgen.env/bin/activate pip install -r scripts/populate_tox/requirements.txt python scripts/populate_tox/populate_tox.py """