Merge pull request #862 from shanehead/feature/pyspark4.1

Support PySpark 4.1.x

Author: mitchelllisle

Makefile

@@ -78,7 +78,7 @@ install-all-pyspark3:
 
 install-all-pyspark4:
 	poetry install --with dev,test,docs
-	poetry run pip install pyspark==4.0.0
+	poetry run pip install pyspark==4.1.1
 
 install-dev-local: ## install all the stuff you need to develop locally
 	pip install --upgrade pip

README.md

@@ -36,7 +36,7 @@ pip install "sparkdantic[pyspark]"
 
 ### Supported PySpark versions
 
-PySpark version `3.3.0` or higher, up to but not including `4.1.0`
+PySpark version `3.3.0` or higher, up to but not including `4.2.0`
 
 ## Usage
 

poetry.lock

@@ -4,12 +4,12 @@ version = "2.7.1"
 description = "A pydantic -> spark schema library"
 authors = ["Mitchell Lisle <m.lisle90@gmail.com>"]
 readme = "README.md"
-packages = [{include = "sparkdantic", from = "src"}]
+packages = [{ include = "sparkdantic", from = "src" }]
 
 [tool.poetry.dependencies]
 python = ">3.9.1,<4.0.0"
 pydantic = "^2.1.1"
-pyspark = { version = ">=3.3.0,<=4.0.2", optional = true }
+pyspark = { version = ">=3.3.0,<=4.2.0", optional = true }
 jmespath = "^1.0.1"
 pyparsing = "^3.1.1"
 packaging = ">=24.2,<27.0"
@@ -35,7 +35,7 @@ mkdocs = "^1.4.2"
 mkdocs-material = "^9.1.4"
 mkdocs-gen-files = ">=0.4,<0.7"
 mkdocs-literate-nav = "^0.6.0"
-mkdocstrings = {extras = ["python"], version = ">=0.20,<0.31"}
+mkdocstrings = { extras = ["python"], version = ">=0.20,<0.31" }
 mkautodoc = "^0.2.0"
 pymdown-extensions = "^10.0.1"
 

pyproject.toml

@@ -4,7 +4,7 @@
 
 MIN_PYSPARK_VERSION = '3.3.0'
 """Inclusive minimum version of PySpark required"""
-MAX_PYSPARK_VERSION = '4.1.0'
+MAX_PYSPARK_VERSION = '4.2.0'
 """Exclusive maximum version of PySpark supported"""
 
 try:

src/sparkdantic/utils.py

@@ -3,7 +3,7 @@
 from pydantic import BaseModel, computed_field
 from pyspark.sql.types import IntegerType, LongType, StringType, StructField, StructType
 
-from sparkdantic import create_spark_schema, SparkField
+from sparkdantic import SparkField, create_spark_schema
 
 
 class ComputedOnlyModel(BaseModel):
@@ -97,7 +97,7 @@ def d(self) -> int:
 
 def test_computed_field_with_spark_type():
     class ComputedWithSparkType(BaseModel):
-        @computed_field(json_schema_extra={"spark_type": LongType})
+        @computed_field(json_schema_extra={'spark_type': LongType})
         @property
         def d(self) -> int:
             return 4
@@ -129,7 +129,7 @@ def d(self) -> Annotated[int, SparkField(spark_type=LongType)]:
 
 def test_computed_field_with_spark_type_over_annotated_return():
     class ComputedWithSparkType(BaseModel):
-        @computed_field(json_schema_extra={"spark_type": LongType})
+        @computed_field(json_schema_extra={'spark_type': LongType})
         @property
         def d(self) -> Annotated[int, SparkField(spark_type=StringType)]:
             return 4

tests/test_computed_field.py

@@ -50,7 +50,8 @@ def test_no_pyspark_raises_import_error(no_pyspark):
         ('3.3.0', False),
         ('3.5.0', False),
         ('4.0.0', False),
-        ('4.1.0', True),
+        ('4.1.0', False),
+        ('4.2.0', True),
     ],
 )
 def test_require_pyspark_version_in_range(