Unpin scipy (#1074)

Author: nasaul

pyproject.toml

@@ -30,8 +30,9 @@ dependencies = [
     "llvmlite>=0.41.0,<0.43.0 ; sys_platform == 'win32' and python_version >= '3.10' and python_version < '3.12'",
     "numpy>=1.21.6",
     "pandas>=1.3.5",
-    "scipy>=1.7.3,<1.16.0",
-    "statsmodels>=0.13.2",
+    "scipy>=1.7.3,<1.14 ; python_version < '3.10'",
+    "scipy>=1.7.3 ; python_version >= '3.10'",
+    "statsmodels>=0.14.5",
     "tqdm",
     "fugue>=0.8.1",
     "utilsforecast>=0.1.4",
@@ -67,14 +68,8 @@ dev = [
     "pip-licenses",
 ]
 dask = ["dask<=2024.12.1", "fugue[dask]>=0.8.1"]
-ray = [
-    "fugue[ray]>=0.8.1 ; python_version < '3.12'",
-    "protobuf>=3.15.3,<4.0.0 ; python_version < '3.12'",
-    "numpy<2 ; python_version < '3.12'",
-    "pandas<2.2 ; python_version < '3.12'",
-    "ray<=2.10 ; python_version < '3.12'",
-]
-spark = ["fugue[spark]>=0.8.1"]
+ray = ["fugue[ray]>=0.9.4", "protobuf>=3.15.3,<4.0.0 ; python_version < '3.12'"]
+spark = ["fugue[spark]>=0.8.1", "pyspark<4.1.0; sys_platform == 'win32'"]
 plotly = ["plotly", "plotly-resampler"]
 polars = ["polars[numpy]"]
 docs = [
@@ -84,8 +79,10 @@ docs = [
 ]
 all = [
     "dask<=2024.12.1",
-    "fugue[dask]>=0.8.1",
-    "fugue[spark]>=0.8.1",
+    "fugue[dask]>=0.9.4",
+    "fugue[spark]>=0.9.4",
+    "fugue[ray]>=0.9.4; sys_platform != 'win32'",
+    "pyspark<4.1.0; sys_platform == 'win32'",
     "plotly",
     "plotly-resampler",
     "polars[numpy]",
@@ -107,11 +104,8 @@ all = [
     "setuptools<70",
     "supersmoother",
     "yfinance",
-    "fugue[ray]>=0.8.1 ; python_version < '3.12'",
     "protobuf>=3.15.3,<4.0.0 ; python_version < '3.12'",
-    "numpy<2 ; python_version < '3.12'",
-    "pandas<2.2 ; python_version < '3.12'",
-    "ray<=2.10 ; python_version < '3.12'",
+    "numpy",
 ]
 
 [tool.setuptools]

tests/test_distributed_fugue.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import pytest
 
-if not sys.version_info >= (3, 12):
+if sys.platform != "win32":
     import ray
 from dask.distributed import Client
 from fugue_dask import DaskExecutionEngine
@@ -185,17 +185,47 @@ def test_distribute_cv_predictions(df):
 
 
 # # | eval: false
+@pytest.fixture(scope="module")
+def ray_session():
+    """Initialize Ray once for all tests in this module and shutdown afterwards."""
+    if sys.platform == "win32":
+        yield None
+        return
+
+    # Initialize Ray with runtime environment to exclude large files
+    if not ray.is_initialized():
+        ray.init(
+            num_cpus=2,
+            ignore_reinit_error=True,
+            include_dashboard=False,
+            _metrics_export_port=None,
+            runtime_env={
+                "working_dir": None,  # Don't upload working directory for local testing
+            },
+        )
+
+    yield ray
+
+    # Cleanup: shutdown Ray after all tests in this module complete
+    if ray.is_initialized():
+        ray.shutdown()
+
+
 @pytest.fixture
-def ray_df():
+def ray_df(ray_session):
+    """Generate test data as Ray Dataset."""
+    if sys.platform == "win32":
+        pytest.skip("Ray is in beta for Windows.")
+
     # Generate Synthetic Panel Data.
-    df = generate_series(10).reset_index()
+    df = generate_series(5).reset_index()
     df["unique_id"] = df["unique_id"].astype(str)
     df = ray.data.from_pandas(df).repartition(2)
     return df
 
 
 @pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="This test is not compatible with Python 3.12+"
+    sys.platform == "win32", reason="Ray is in beta for Windows."
 )
 def test_ray_cv_predictions(ray_df):
     df = ray_df
@@ -231,7 +261,7 @@ def test_ray_cv_predictions(ray_df):
 
 
 @pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="This test is not compatible with Python 3.12+"
+    sys.platform == "win32", reason="Ray is in beta for Windows."
 )
 def test_ray_cv_fallback_model(ray_df):
     df = ray_df
@@ -249,7 +279,7 @@ def test_ray_cv_fallback_model(ray_df):
     pd.testing.assert_frame_equal(fcst_fugue.astype(fcst_stats.dtypes), fcst_stats)
 
 @pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="This test is not compatible with Python 3.12+"
+    sys.platform == "win32", reason="Ray is in beta for Windows."
 )
 def test_ray_distributed_exogenous_regressors(df_w_ex):
     df_w_ex, train_df, test_df, xreg = df_w_ex