chore(spark): bump version to 4.0.1 (#304)

* chore(spark): bump version to 3.5.7

* format command args consistently

* update to spark 4.0.1

Author: razvan

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -144,13 +144,13 @@ data:
         stackable.tech/vendor: Stackable
     spec:
       sparkImage:
-        productVersion: 3.5.6
+        productVersion: 4.0.1
       mode: cluster
       mainApplicationFile: local:///stackable/spark/jobs/spark-ingest-into-lakehouse.py
       deps:
         packages:
-          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1
-          - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.6
+          - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0
+          - org.apache.spark:spark-sql-kafka-0-10_2.13:4.0.1
       s3connection:
         reference: minio
       sparkConf:

demos/end-to-end-security/create-spark-report.yaml

@@ -55,12 +55,12 @@ data:
       name: spark-report
     spec:
       sparkImage:
-        productVersion: 3.5.6
+        productVersion: 4.0.1
       mode: cluster
       mainApplicationFile: local:///stackable/spark/jobs/spark-report.py
       deps:
         packages:
-          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1
+          - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0
       sparkConf:
         spark.driver.extraClassPath: /stackable/config/hdfs
         spark.executor.extraClassPath: /stackable/config/hdfs

demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml

@@ -10,22 +10,22 @@ spec:
         - name: wait-for-testdata
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
-          - bash
-          - -euo
-          - pipefail
-          - -c
-          - |
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
               echo 'Waiting for job load-ny-taxi-data to finish'
               kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data
       containers:
         - name: create-spark-anomaly-detection-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
-          - bash
-          - -euo
-          - pipefail
-          - -c
-          - |
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
               echo 'Submitting Spark job'
               kubectl apply -f /tmp/manifest/spark-ad-job.yaml
           volumeMounts:
@@ -51,12 +51,12 @@ data:
       name: spark-ad
     spec:
       sparkImage:
-        productVersion: 3.5.6
+        productVersion: 4.0.1
       mode: cluster
       mainApplicationFile: local:///spark-scripts/spark-ad.py
       deps:
         packages:
-          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1
+          - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.1
         requirements:
           - scikit-learn==1.4.0
       s3connection:

stacks/airflow/airflow.yaml

@@ -303,7 +303,7 @@ data:
     spec:
       version: "1.0"
       sparkImage:
-        productVersion: 3.5.6
+        productVersion: 4.0.1
       mode: cluster
       mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
       job:
@@ -331,6 +331,8 @@ data:
             memory:
               limit: 1024Mi
         replicas: 3
+
+
 # {% endraw %}
 ---
 apiVersion: v1

stacks/jupyterhub-pyspark-hdfs/jupyterlab.yaml

@@ -21,7 +21,7 @@ spec:
       serviceAccountName: default
       containers:
         - name: jupyterlab
-          image: oci.stackable.tech/stackable/spark-connect-client:3.5.6-stackable0.0.0-dev
+          image: oci.stackable.tech/stackable/spark-connect-client:4.0.1-stackable0.0.0-dev
           imagePullPolicy: IfNotPresent
           command:
             - bash
@@ -39,8 +39,12 @@ spec:
               name: notebook
       initContainers:
         - name: download-notebook
-          image: oci.stackable.tech/stackable/spark-connect-client:3.5.6-stackable0.0.0-dev
-          command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb']
+          image: oci.stackable.tech/stackable/spark-connect-client:4.0.1-stackable0.0.0-dev
+          command:
+            - bash
+          args:
+            - -c
+            - curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb
           volumeMounts:
             - mountPath: /notebook
               name: notebook

stacks/jupyterhub-pyspark-hdfs/notebook.ipynb

@@ -53,7 +53,7 @@
     "#\n",
     "# See: https://issues.apache.org/jira/browse/SPARK-46032\n",
     "#\n",
-    "spark.addArtifacts(\"/stackable/spark/connect/spark-connect_2.12-3.5.6.jar\")"
+    "spark.addArtifacts(\"/stackable/spark/connect/spark-connect-4.0.1.jar\")"
    ]
   },
   {

stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml

@@ -30,8 +30,8 @@ spec:
   image:
     # Using an image that includes scikit-learn (among other things)
     # because this package needs to be available on the executors.
-    custom: oci.stackable.tech/stackable/spark-connect-client:3.5.6-stackable0.0.0-dev
-    productVersion: 3.5.6
+    custom: oci.stackable.tech/stackable/spark-connect-client:4.0.1-stackable0.0.0-dev
+    productVersion: 4.0.1
     pullPolicy: IfNotPresent
   args:
   server: