Pin PySpark due to test failures/incompatibilities (#2010)

deepyaman · web-flow · commit 5fdd8eb090d4 · 2025-05-27T14:05:02.000-04:00
Signed-off-by: Deepyaman Datta &lt;deepyaman.datta@utexas.edu&gt;
diff --git a/environment.yml b/environment.yml
@@ -23,7 +23,7 @@ dependencies:
   - pandas-stubs
 
   # pyspark extra
-  - pyspark[connect] >= 3.2.0
+  - pyspark[connect] >= 3.2.0, < 4.0.0
 
   # polars extra
   - polars >= 0.20.0
diff --git a/pyproject.toml b/pyproject.toml
@@ -64,7 +64,7 @@ geopandas = [
     "geopandas",
     "shapely",
 ]
-pyspark = ["pyspark[connect] >= 3.2.0"]
+pyspark = ["pyspark[connect] >= 3.2.0, < 4.0.0"]
 modin = [
     "modin",
     "ray",
@@ -92,7 +92,7 @@ all = [
     "pyyaml >= 5.1",
     "black",
     "frictionless <= 4.40.8",
-    "pyspark[connect] >= 3.2.0",
+    "pyspark[connect] >= 3.2.0, < 4.0.0",
     "modin",
     "ray",
     "dask[dataframe]",
diff --git a/requirements.txt b/requirements.txt
@@ -12,7 +12,7 @@ pyarrow
 pydantic
 scipy
 pandas-stubs
-pyspark[connect] >= 3.2.0
+pyspark[connect] >= 3.2.0, < 4.0.0
 polars >= 0.20.0
 modin
 protobuf
