Merge pull request #295 from ing-bank/test_spark4_scala213

Use spark4 with scala213

Author: mbaak

.github/workflows/build.yml

@@ -52,6 +52,12 @@ jobs:
             python: "3.12"
             os: ubuntu-latest
             dependency_constraints: '"pandas>=2" "numpy>=2"'
+          - SPARK_VERSION: "4.0.0"
+            HADOOP_VERSION: "3"
+            JAVA_VERSION: "17"
+            python: "3.12"
+            os: ubuntu-latest
+            dependency_constraints: '"pandas>=2" "numpy>=2"'
     runs-on: ${{ matrix.os }}
     name: ${{ matrix.os }}, Spark ${{ matrix.SPARK_VERSION}}, Python ${{ matrix.python }}
 

README.rst

@@ -17,26 +17,27 @@ It works with numerical, ordinal, categorical features, and the histograms can b
 as trends, shifts, peaks, outliers, anomalies, changing correlations, etc,
 using monitoring business rules.
 
+Latest update: Sep 2025.
+
 |example|
 
 |histograms|
 
-Announcements
-=============
-
-Spark 3.0
----------
+Spark
+=====
 
-With Spark 3.0, based on Scala 2.12, make sure to pick up the correct `histogrammar` jar files:
+For Spark make sure to pick up the correct `histogrammar` jar files. Spark 4.X is based on Scala 2.13; Spark 3.X is based on Scala 2.12 or 2.13.
 
 .. code-block:: python
 
   spark = SparkSession.builder.config(
       "spark.jars.packages",
-      "io.github.histogrammar:histogrammar_2.12:1.0.20,io.github.histogrammar:histogrammar-sparksql_2.12:1.0.20",
+      "io.github.histogrammar:histogrammar_2.13:1.0.30,io.github.histogrammar:histogrammar-sparksql_2.13:1.0.30",
   ).getOrCreate()
 
-For Spark 2.X compiled against scala 2.11, in the string above simply replace 2.12 with 2.11.
+
+For Scala 2.12, in the string above simply replace "2.13" with "2.12" (twice).
+
 
 Examples
 ========

popmon/notebooks/popmon_tutorial_advanced.ipynb

@@ -175,9 +175,9 @@
    "outputs": [],
    "source": [
     "if pyspark_installed:\n",
-    "    scala = \"2.12\" if int(pyspark_version[0]) >= 3 else \"2.11\"\n",
-    "    hist_jar = f\"io.github.histogrammar:histogrammar_{scala}:1.0.20\"\n",
-    "    hist_spark_jar = f\"io.github.histogrammar:histogrammar-sparksql_{scala}:1.0.20\"\n",
+    "    scala = \"2.12\" if int(pyspark_version[0]) == 3 else \"2.13\"\n",
+    "    hist_jar = f\"io.github.histogrammar:histogrammar_{scala}:1.0.30\"\n",
+    "    hist_spark_jar = f\"io.github.histogrammar:histogrammar-sparksql_{scala}:1.0.30\"\n",
     "\n",
     "    spark = SparkSession.builder.config(\n",
     "        \"spark.jars.packages\", f\"{hist_spark_jar},{hist_jar}\"\n",

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "numpy>=1.18.0",
     "pandas>=0.25.1",
     "scipy>=1.5.2",
-    "histogrammar>=1.1.1",
+    "histogrammar>=1.1.2",
     "phik",
     "jinja2",
     "tqdm",

tests/popmon/spark/jars/histogrammar-sparksql_2.12-1.0.30.jar

@@ -23,9 +23,9 @@ def spark_context():
 
     current_path = Path(__file__).parent
 
-    scala = "2.12" if int(pyspark_version[0]) >= 3 else "2.11"
-    hist_spark_jar = current_path / f"jars/histogrammar-sparksql_{scala}-1.0.11.jar"
-    hist_jar = current_path / f"jars/histogrammar_{scala}-1.0.11.jar"
+    scala = "2.12" if int(pyspark_version[0]) == 3 else "2.13"
+    hist_spark_jar = current_path / f"jars/histogrammar-sparksql_{scala}-1.0.30.jar"
+    hist_jar = current_path / f"jars/histogrammar_{scala}-1.0.30.jar"
 
     spark = (
         SparkSession.builder.master("local")
@@ -115,10 +115,10 @@ def test_spark_make_histograms(spark_context):
     # backwards compatibility
     for name in names:
         v1 = deepcopy(getattr(pytest, name))
-        v1["data"]["name"] = f"'{name}'"
+        v1["data"]["name"] = f"{name}"
 
         v2 = deepcopy(getattr(pytest, name))
-        v2["data"]["name"] = f"b'{name}'"
+        v2["data"]["name"] = f"b{name}"
 
         output = current_hists[name].toJson()
         assert output in (v1, v2)

tests/popmon/spark/jars/histogrammar-sparksql_2.13-1.0.30.jar

@@ -22,9 +22,9 @@ def spark_context():
 
     current_path = Path(__file__).parent
 
-    scala = "2.12" if int(pyspark_version[0]) >= 3 else "2.11"
-    hist_spark_jar = current_path / f"jars/histogrammar-sparksql_{scala}-1.0.11.jar"
-    hist_jar = current_path / f"jars/histogrammar_{scala}-1.0.11.jar"
+    scala = "2.12" if int(pyspark_version[0]) == 3 else "2.13"
+    hist_spark_jar = current_path / f"jars/histogrammar-sparksql_{scala}-1.0.30.jar"
+    hist_jar = current_path / f"jars/histogrammar_{scala}-1.0.30.jar"
 
     spark = (
         SparkSession.builder.master("local")