fix: spark configuration

Author: reinthal

nessie-stack/Dockerfile

@@ -1,4 +1,4 @@
-FROM jupyter/pyspark-notebook:latest
+FROM docker.io/jupyter/pyspark-notebook:latest
 
 USER root
 
@@ -11,14 +11,13 @@ RUN wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime
     wget https://repo1.maven.org/maven2/org/projectnessie/nessie-integrations/nessie-spark-extensions-3.5_2.12/0.102.5/nessie-spark-extensions-3.5_2.12-0.102.5.jar -P /usr/local/spark/jars/
 
 # Install Python packages (update pyiceberg to match JAR version)
-RUN pip install pyiceberg==0.9.0 pynessie==0.67.0
+RUN pip install pyiceberg==0.9.0 pynessie==0.67.0 findspark
 
 # Create a directory for init scripts
 RUN mkdir -p /usr/local/bin/start-notebook.d
 
 # Create init script to configure Spark environment
-RUN echo '#!/bin/bash\nexport PYSPARK_DRIVER_PYTHON=jupyter\nexport PYSPARK_DRIVER_PYTHON_OPTS="lab --NotebookApp.token=\'\' --NotebookApp.password=\'\'"' > /usr/local/bin/start-notebook.d/spark-config.sh && \
-    chmod +x /usr/local/bin/start-notebook.d/spark-config.sh
+COPY spark-config.sh /usr/local/bin/start-notebook.d/spark-config.sh
 
 USER $NB_UID
 

nessie-stack/config_notebook.sh

@@ -120,11 +120,10 @@ services:
         --hiveconf hive.server2.thrift.port=10000 \
         --hiveconf hive.server2.thrift.bind.host=0.0.0.0 \
         --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
-        --conf spark.sql.catalog.iceberg=org.apache.iceberg.spark.SparkCatalog \
-        --conf spark.sql.catalog.iceberg.type=nessie \
+        --conf spark.sql.catalog.iceberg=org.apache.iceberg.spark.SparkSessionCatalog \
+        --conf spark.sql.catalog.iceberg.type=NessieCatalog \
         --conf spark.sql.catalog.iceberg.uri=http://nessie:19120/api/v1 \
         --conf spark.sql.catalog.iceberg.ref=main \
-        --conf spark.sql.catalog.iceberg.catalog-impl=org.apache.iceberg.aws.s3.S3FileIO \
         --conf spark.sql.catalog.iceberg.s3.endpoint=http://minio:9000 \
         --conf spark.sql.catalog.iceberg.s3.path-style-access=true \
         --conf spark.sql.catalog.iceberg.warehouse=s3a://nessie \

nessie-stack/docker-compose.yml

@@ -0,0 +1,3 @@
+#!/bin/bash
+export PYSPARK_DRIVER_PYTHON=jupyter
+export PYSPARK_DRIVER_PYTHON_OPTS="lab --NotebookApp.token='' --NotebookApp.password=''"