From 6748c98778e97c72942b77f50243e82a2f852350 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Mon, 7 Apr 2025 16:27:47 -0700 Subject: [PATCH 1/2] fix & better align nox against GitHub Actions CI --- noxfile.py | 203 +++++++++++++++++++++++++----- utils/generate_random_versions.py | 22 ++-- 2 files changed, 181 insertions(+), 44 deletions(-) diff --git a/noxfile.py b/noxfile.py index 187f66bfa8..8914d933ab 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,5 +1,6 @@ from __future__ import annotations +from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING import nox @@ -8,60 +9,162 @@ from nox.sessions import Session nox.options.default_venv_backend = "uv" -nox.options.reuse_venv = "yes" +nox.options.reuse_venv = "never" -PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] +PYTHON_VERSIONS = { + "pytest": ["3.8", "3.10", "3.11", "3.12", "3.13"], + "random": ["3.9"], + "minimum": "3.8", + "pretty_old": "3.8", + "not_so_old": "3.10", + "nightly": "3.13", +} -def run_common(session: Session, coverage_threshold: float) -> None: - if session.python == "3.8": - session.install("-e . --group dev-core") - elif session.python == "3.12": - session.install("-e .[dask,modin] --group dev-core --group extra") - else: - session.install("-e .[dask,modin,pyspark,ibis] --group dev-core --group extra") +@nox.session(python=PYTHON_VERSIONS["pytest"]) +def pytest_coverage(session: Session) -> None: + pytest_cmd = [ + "pytest", + "tests", + "--cov=narwhals", + "--cov=tests", + ] + if session.python == "3.8": + session.install( + "-e", ".[pandas,polars,pyarrow]", "backports.zoneinfo", "--group", "tests" + ) + session.run( + *pytest_cmd, + "--cov-fail-under=80", + "--constructors", + "pandas,pyarrow,polars[eager],polars[lazy]", + ) + + elif session.python in {"3.10", "3.12"}: + session.install( + "-e", ".[dask,modin]", "--group", "core-tests", "--group", "extra" + ) + session.run( + *pytest_cmd, + "--cov-fail-under=95", + "--runslow", + "--constructors", + "pandas,pandas[nullable],pandas[pyarrow],pyarrow,modin[pyarrow],polars[eager],polars[lazy],dask,duckdb,sqlframe", + ) + + elif session.python in {"3.11", "3.13"}: + session.install( + "-e", ".[modin, dask]", "--group", "core-tests", "--group", "extra" + ) + session.install("-U", "--pre", "duckdb") + if session.python != "3.13": + with NamedTemporaryFile() as f: + f.write(b"setuptools<78\n") + session.install("-b", f.name, "-e", ".[pyspark]") + + if session.python == "3.11": + session.install("-e", ".[ibis]") + + session.run( + *pytest_cmd, + "--cov-fail-under=100", + "--runslow", + "--all-cpu-constructors", + env={ + "NARWHALS_POLARS_NEW_STREAMING": str(session.python == "3.11"), + }, + ) + + if session.python == PYTHON_VERSIONS["pytest"][-1]: + session.run("pytest", "narwhals/", "--doctest-modules") + + +@nox.session(python=PYTHON_VERSIONS["minimum"]) +def minimum(session: Session) -> None: + session.install( + "pandas==0.25.3", + "polars==0.20.3", + "numpy==1.17.5", + "pyarrow==11.0.0", + "pyarrow-stubs<17", + "scipy==1.5.0", + "scikit-learn==1.1.0", + "duckdb==1.0", + "tzdata", + "backports.zoneinfo", + ) + session.install("-e", ".", "--group", "tests") session.run( "pytest", "tests", "--cov=narwhals", "--cov=tests", - f"--cov-fail-under={coverage_threshold}", + "--cov-fail-under=50", "--runslow", + "--constructors=pandas,pyarrow,polars[eager],polars[lazy]", ) - if session.python == "3.12": - session.run("pytest", "narwhals", "--doctest-modules") - -@nox.session(python=PYTHON_VERSIONS) # type: ignore[misc] -def pytest_coverage(session: Session) -> None: - coverage_threshold = 85 if session.python == "3.8" else 100 - - run_common(session, coverage_threshold) - - -@nox.session(python=PYTHON_VERSIONS[0]) # type: ignore[misc] -@nox.parametrize("pandas_version", ["0.25.3", "1.1.5"]) # type: ignore[misc] -def min_and_old_versions(session: Session, pandas_version: str) -> None: +@nox.session(python=PYTHON_VERSIONS["pretty_old"]) +def pretty_old(session: Session) -> None: session.install( - f"pandas=={pandas_version}", + "pandas==1.1.5", "polars==0.20.3", "numpy==1.17.5", "pyarrow==11.0.0", + "pyarrow-stubs<17", "scipy==1.5.0", "scikit-learn==1.1.0", + "duckdb==1.0", + "tzdata", + "backports.zoneinfo", + ) + session.install("-e", ".", "--group", "tests") + session.run( + "pytest", + "tests", + "--cov=narwhals", + "--cov=tests", + "--cov-fail-under=50", + "--runslow", + "--constructors=pandas,pyarrow,polars[eager],polars[lazy]", + ) + + +@nox.session(python=PYTHON_VERSIONS["not_so_old"]) +def not_so_old(session: Session) -> None: + session.install( + "pandas==2.0.3", + "polars==0.20.8", + "numpy==1.24.4", + "pyarrow==15.0.0", + "pyarrow-stubs<17", + "scipy==1.8.0", + "scikit-learn==1.3.0", + "duckdb==1.0", + "dask[dataframe]==2024.10 ", "tzdata", ) - if pandas_version == "1.1.5": - session.install("pyspark==3.3.0") - run_common(session, coverage_threshold=50) + session.install("-e", ".", "--group", "tests") + session.run( + "pytest", + "tests", + "--cov=narwhals", + "--cov=tests", + "--cov-fail-under=50", + "--runslow", + "--constructors=pandas,pyarrow,polars[eager],polars[lazy]", + ) -@nox.session(python=PYTHON_VERSIONS[-1]) # type: ignore[misc] +@nox.session(python=PYTHON_VERSIONS["nightly"]) def nightly_versions(session: Session) -> None: - session.install("polars") + session.install("-e", ".", "--group", "tests") + session.install("--pre", "polars") + + session.run("pip", "uninstall", "pandas", "--yes") session.install( # pandas nightly "--pre", "--extra-index-url", @@ -69,6 +172,12 @@ def nightly_versions(session: Session) -> None: "pandas", ) + session.run("pip", "uninstall", "pyarrow", "--yes") + session.install( # pyarrow nightly + "--extra-index-url", "https://pypi.fury.io/arrow-nightlies/", "--pre", "pyarrow" + ) + + session.run("pip", "uninstall", "numpy", "--yes") session.install( # numpy nightly "--pre", "--extra-index-url", @@ -76,12 +185,42 @@ def nightly_versions(session: Session) -> None: "numpy", ) - session.run("uv", "pip", "install", "pip") session.run( # dask nightly "pip", "install", "git+https://github.com/dask/distributed", "git+https://github.com/dask/dask", - "git+https://github.com/dask/dask-expr", ) - run_common(session, coverage_threshold=50) + + session.install("-U", "--pre", "duckdb") # duckdb nightly + + session.run( + "pytest", + "tests", + "--cov=narwhals", + "--cov=tests", + "--cov-fail-under=50", + "--runslow", + "--constructors=pandas,pandas[nullable],pandas[pyarrow],pyarrow,polars[eager],polars[lazy],dask,duckdb", + ) + + +@nox.session(python=PYTHON_VERSIONS["random"]) +def random(session: Session) -> None: + from utils.generate_random_versions import requirements + + with NamedTemporaryFile("w", suffix=".txt") as fd: + fd.write(requirements) + fd.flush() + + session.install("-r", fd.name) + session.install("-e", ".", "--group", "tests") + + session.run( + "pytest", + "tests", + "--cov=narwhals", + "--cov=tests", + "--cov-fail-under=80", + "--constructors=pandas,pyarrow,polars[eager],polars[lazy]", + ) diff --git a/utils/generate_random_versions.py b/utils/generate_random_versions.py index 7ad8e044d6..f2682db24e 100644 --- a/utils/generate_random_versions.py +++ b/utils/generate_random_versions.py @@ -1,6 +1,7 @@ from __future__ import annotations import random +from textwrap import dedent PANDAS_AND_NUMPY_VERSION = [ # ("1.0.5", "1.18.5"), # fails to build in CI # noqa: ERA001 @@ -57,16 +58,13 @@ pandas_version, numpy_version = random.choice(PANDAS_AND_NUMPY_VERSION) polars_version = random.choice(POLARS_VERSION) pyarrow_version = random.choice(PYARROW_VERSION) +requirements = dedent(f""" + pandas=={pandas_version} + numpy=={numpy_version} + polars=={polars_version} + pyarrow=={pyarrow_version} +""").lstrip() -content = f"pandas=={pandas_version}\nnumpy=={numpy_version}\npolars=={polars_version}\npyarrow=={pyarrow_version}\n" -with open("random-requirements.txt", "w") as fd: - fd.write(content) - -with open("pyproject.toml") as fd: - content = fd.read() -content = content.replace( - 'filterwarnings = [\n "error",\n]', - "filterwarnings = [\n \"error\",\n 'ignore:distutils Version classes are deprecated:DeprecationWarning',\n]", -) -with open("pyproject.toml", "w") as fd: - fd.write(content) +if __name__ == "__main__": + with open("random-requirements.txt", "w") as fd: + fd.write(requirements) From 6b58ed34e30893d0c667e6b43d12de0e328eb787 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Tue, 8 Apr 2025 11:09:23 -0700 Subject: [PATCH 2/2] ensure pip is installed where it is directly invoked --- noxfile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 8914d933ab..6273924ead 100644 --- a/noxfile.py +++ b/noxfile.py @@ -61,6 +61,7 @@ def pytest_coverage(session: Session) -> None: if session.python != "3.13": with NamedTemporaryFile() as f: f.write(b"setuptools<78\n") + f.flush() session.install("-b", f.name, "-e", ".[pyspark]") if session.python == "3.11": @@ -159,7 +160,8 @@ def not_so_old(session: Session) -> None: @nox.session(python=PYTHON_VERSIONS["nightly"]) -def nightly_versions(session: Session) -> None: +def nightly(session: Session) -> None: + session.install("pip") session.install("-e", ".", "--group", "tests") session.install("--pre", "polars") @@ -185,6 +187,7 @@ def nightly_versions(session: Session) -> None: "numpy", ) + session.run("pip", "uninstall", "dask", "dask-expr", "--yes") session.run( # dask nightly "pip", "install",