feat: support for Spark 4

Author: razvan

rust/operator-binary/src/connect/server.rs

@@ -551,20 +551,21 @@ pub(crate) fn server_properties(
         ),
         (
             "spark.driver.extraClassPath".to_string(),
-            Some(format!("/stackable/spark/extra-jars/*:/stackable/spark/connect/spark-connect_2.12-{spark_version}.jar")),
+            Some(format!("/stackable/spark/extra-jars/*:/stackable/spark/connect/spark-connect-{spark_version}.jar")),
         ),
         (
             "spark.metrics.conf".to_string(),
-            Some(format!("{VOLUME_MOUNT_PATH_CONFIG}/{METRICS_PROPERTIES_FILE}")),
+            Some(format!(
+                "{VOLUME_MOUNT_PATH_CONFIG}/{METRICS_PROPERTIES_FILE}"
+            )),
         ),
         // This enables the "/metrics/executors/prometheus" endpoint on the server pod.
         // The driver collects metrics from the executors and makes them available here.
-        // The "/metrics/prometheus" endpoint delievers the driver metrics.
+        // The "/metrics/prometheus" endpoint delivers the driver metrics.
         (
             "spark.ui.prometheus.enabled".to_string(),
             Some("true".to_string()),
         ),
-
     ]
     .into();
 

tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2

@@ -8,11 +8,11 @@ spec:
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
   sparkImage:
-{% if test_scenario['values']['spark'].find(",") > 0 %}
-    custom: "{{ test_scenario['values']['spark'].split(',')[1] }}"
-    productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}"
+{% if test_scenario['values']['spark-hbase-connector'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['spark-hbase-connector'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['spark-hbase-connector'].split(',')[0] }}"
 {% else %}
-    productVersion: "{{ test_scenario['values']['spark'] }}"
+    productVersion: "{{ test_scenario['values']['spark-hbase-connector'] }}"
 {% endif %}
     # pullPolicy: IfNotPresent
     pullPolicy: Always

tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2

@@ -57,10 +57,10 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: simple-connect-app
-{% if test_scenario['values']['spark-connect-client'].find(",") > 0 %}
-          image: "{{ test_scenario['values']['spark-connect-client'].split(',')[1] }}"
+{% if test_scenario['values']['spark-connect'].find(",") > 0 %}
+          image: "{{ test_scenario['values']['spark-connect'].split(',')[1] }}"
 {% else %}
-          image: oci.stackable.tech/stackable/spark-connect-client:{{ test_scenario['values']['spark-connect-client'] }}-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/spark-connect-client:{{ test_scenario['values']['spark-connect'] }}-stackable0.0.0-dev
 {% endif %}
           imagePullPolicy: IfNotPresent
           command:

tests/test-definition.yaml

@@ -7,20 +7,22 @@ dimensions:
     values:
       - 3.5.5
       - 3.5.6
+      - 4.0.0
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
       # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
-  - name: spark-delta-lake
+  - name: spark-hbase-connector
     values:
+      - 3.5.5
       - 3.5.6
-      # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
-  - name: spark-connect
+  - name: spark-delta-lake
     values:
       - 3.5.6
       # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
-  - name: spark-connect-client
+  - name: spark-connect
     values:
       - 3.5.6
+      - 4.0.0
       # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
   - name: hbase
     values:
@@ -99,7 +101,7 @@ tests:
       - openshift
   - name: hbase-connector
     dimensions:
-      - spark
+      - spark-hbase-connector
       - hbase
       - hdfs-latest
       - zookeeper-latest
@@ -113,7 +115,6 @@ tests:
   - name: spark-connect
     dimensions:
       - spark-connect
-      - spark-connect-client
       - openshift
 
 suites: