trying something

Author: sentrivana

.github/workflows/test-integrations-tasks.yml

@@ -46,6 +46,11 @@ jobs:
           allow-prereleases: true
       - name: Start Redis
         uses: supercharge/[email protected]
+      - name: Install Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
       - name: Setup Test Env
         run: |
           pip install "coverage[toml]" tox
@@ -135,6 +140,11 @@ jobs:
           allow-prereleases: true
       - name: Start Redis
         uses: supercharge/[email protected]
+      - name: Install Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
       - name: Setup Test Env
         run: |
           pip install "coverage[toml]" tox

scripts/populate_tox/config.py

@@ -163,10 +163,6 @@
     "spark": {
         "package": "pyspark",
         "python": ">=3.8",
-        # XXX: spark 4.0.0 needs a different java version installed
-        # need to find a way to activate different java versions for <4.0
-        # and >=4.0
-        "include": "!=4.0.0",
     },
     "sqlalchemy": {
         "package": "sqlalchemy",

scripts/populate_tox/tox.jinja

@@ -363,6 +363,7 @@ setenv =
     py3.6: COVERAGE_RCFILE=.coveragerc36
 
     django: DJANGO_SETTINGS_MODULE=tests.integrations.django.myapp.settings
+    spark-v{3.0.3,3.5.6}: JAVA_HOME=/usr/lib/jvm/temurin-11-jdk-amd64
 
     common: TESTPATH=tests
     gevent: TESTPATH=tests

scripts/split_tox_gh_actions/split_tox_gh_actions.py

@@ -47,6 +47,10 @@
     "aws_lambda",
 }
 
+FRAMEWORKS_NEEDING_JAVA = {
+    "spark",
+}
+
 # Frameworks grouped here will be tested together to not hog all GitHub runners.
 # If you add or remove a group, make sure to git rm the generated YAML file as
 # well.
@@ -288,6 +292,7 @@ def render_template(group, frameworks, py_versions_pinned, py_versions_latest):
         "needs_docker": bool(set(frameworks) & FRAMEWORKS_NEEDING_DOCKER),
         "needs_postgres": bool(set(frameworks) & FRAMEWORKS_NEEDING_POSTGRES),
         "needs_redis": bool(set(frameworks) & FRAMEWORKS_NEEDING_REDIS),
+        "needs_java": bool(set(frameworks) & FRAMEWORKS_NEEDING_JAVA),
         "py_versions": {
             category: [f'"{version}"' for version in _normalize_py_versions(versions)]
             for category, versions in py_versions.items()

scripts/split_tox_gh_actions/templates/test_group.jinja

@@ -48,6 +48,7 @@
         with:
           python-version: {% raw %}${{ matrix.python-version }}{% endraw %}
           allow-prereleases: true
+
       {% if needs_clickhouse %}
       - name: "Setup ClickHouse Server"
         uses: getsentry/[email protected]
@@ -58,6 +59,14 @@
         uses: supercharge/[email protected]
       {% endif %}
 
+      {% if needs_java %}
+      - name: Install Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+      {% endif %}
+
       - name: Setup Test Env
         run: |
           pip install "coverage[toml]" tox

tox.ini

@@ -10,7 +10,7 @@
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
 #
-# Last generated: 2025-06-03T13:18:52.238209+00:00
+# Last generated: 2025-06-03T13:53:53.259798+00:00
 
 [tox]
 requires =
@@ -230,9 +230,8 @@ envlist =
     {py3.6,py3.11,py3.12}-huey-v2.5.3
 
     {py3.8,py3.9}-spark-v3.0.3
-    {py3.8,py3.9}-spark-v3.2.4
-    {py3.8,py3.10,py3.11}-spark-v3.4.4
     {py3.8,py3.10,py3.11}-spark-v3.5.6
+    {py3.9,py3.12,py3.13}-spark-v4.0.0
 
 
     # ~~~ Web 1 ~~~
@@ -622,9 +621,8 @@ deps =
     huey-v2.5.3: huey==2.5.3
 
     spark-v3.0.3: pyspark==3.0.3
-    spark-v3.2.4: pyspark==3.2.4
-    spark-v3.4.4: pyspark==3.4.4
     spark-v3.5.6: pyspark==3.5.6
+    spark-v4.0.0: pyspark==4.0.0
 
 
     # ~~~ Web 1 ~~~
@@ -776,6 +774,7 @@ setenv =
     py3.6: COVERAGE_RCFILE=.coveragerc36
 
     django: DJANGO_SETTINGS_MODULE=tests.integrations.django.myapp.settings
+    spark-v{3.0.3,3.5.6}: JAVA_HOME=/usr/lib/jvm/temurin-11-jdk-amd64
 
     common: TESTPATH=tests
     gevent: TESTPATH=tests