pyspark

Author: sentrivana

.github/workflows/test-integrations-tasks.yml

@@ -133,10 +133,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
-      - uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: '21'
       - name: Start Redis
         uses: supercharge/[email protected]
       - name: Setup Test Env
@@ -176,9 +172,6 @@ jobs:
       - name: Test spark pinned
         run: |
           set -x # print commands that are executed
-          echo "java home $JAVA_HOME"
-          java --version
-          update-alternatives --list java
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-spark"
       - name: Generate coverage XML (Python 3.6)
         if: ${{ !cancelled() && matrix.python-version == '3.6' }}

scripts/populate_tox/config.py

@@ -163,6 +163,10 @@
     "spark": {
         "package": "pyspark",
         "python": ">=3.8",
+        # XXX: spark 4.0.0 needs a different java version installed
+        # need to find a way to activate different java versions for <4.0
+        # and >=4.0
+        "include": "!=4.0.0",
     },
     "sqlalchemy": {
         "package": "sqlalchemy",

tox.ini

@@ -10,7 +10,7 @@
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
 #
-# Last generated: 2025-06-03T11:56:46.072665+00:00
+# Last generated: 2025-06-03T13:18:52.238209+00:00
 
 [tox]
 requires =
@@ -230,8 +230,9 @@ envlist =
     {py3.6,py3.11,py3.12}-huey-v2.5.3
 
     {py3.8,py3.9}-spark-v3.0.3
+    {py3.8,py3.9}-spark-v3.2.4
+    {py3.8,py3.10,py3.11}-spark-v3.4.4
     {py3.8,py3.10,py3.11}-spark-v3.5.6
-    {py3.9,py3.12,py3.13}-spark-v4.0.0
 
 
     # ~~~ Web 1 ~~~
@@ -621,8 +622,9 @@ deps =
     huey-v2.5.3: huey==2.5.3
 
     spark-v3.0.3: pyspark==3.0.3
+    spark-v3.2.4: pyspark==3.2.4
+    spark-v3.4.4: pyspark==3.4.4
     spark-v3.5.6: pyspark==3.5.6
-    spark-v4.0.0: pyspark==4.0.0
 
 
     # ~~~ Web 1 ~~~