feat: update configs

Author: reinthal

nessie-stack/Dockerfile

@@ -0,0 +1,27 @@
+FROM jupyter/pyspark-notebook:latest
+
+USER root
+
+# Install AWS Hadoop integration for S3
+RUN wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar -P /usr/local/spark/jars/ && \
+    wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar -P /usr/local/spark/jars/
+
+# Install Iceberg dependencies for Spark 3.5 (version 1.8.1)
+RUN wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.8.1/iceberg-spark-runtime-3.5_2.12-1.8.1.jar -P /usr/local/spark/jars/ && \
+    wget https://repo1.maven.org/maven2/org/projectnessie/nessie-integrations/nessie-spark-extensions-3.5_2.12/0.102.5/nessie-spark-extensions-3.5_2.12-0.102.5.jar -P /usr/local/spark/jars/
+
+# Install Python packages (update pyiceberg to match JAR version)
+RUN pip install pyiceberg==0.9.0 pynessie==0.67.0
+
+# Create a directory for init scripts
+RUN mkdir -p /usr/local/bin/start-notebook.d
+
+# Create init script to configure Spark environment
+RUN echo '#!/bin/bash\nexport PYSPARK_DRIVER_PYTHON=jupyter\nexport PYSPARK_DRIVER_PYTHON_OPTS="lab --NotebookApp.token=\'\' --NotebookApp.password=\'\'"' > /usr/local/bin/start-notebook.d/spark-config.sh && \
+    chmod +x /usr/local/bin/start-notebook.d/spark-config.sh
+
+USER $NB_UID
+
+# Create a default spark-defaults.conf
+RUN mkdir -p $HOME/.sparkmagic
+COPY --chown=$NB_UID:$NB_GID spark-defaults.conf /usr/local/spark/conf/spark-defaults.conf

nessie-stack/config_notebook.sh

@@ -89,8 +89,8 @@ services:
     environment:
       - SPARK_MODE=worker
       - SPARK_MASTER_URL=spark://spark-master:7077
-      - SPARK_WORKER_MEMORY=1G
-      - SPARK_WORKER_CORES=1
+      - SPARK_WORKER_MEMORY=2G
+      - SPARK_WORKER_CORES=2
       - SPARK_RPC_AUTHENTICATION_ENABLED=no
       - SPARK_RPC_ENCRYPTION_ENABLED=no
       - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
@@ -141,7 +141,8 @@ services:
       - app-network
 
   jupyter:
-    image: jupyter/pyspark-notebook:latest
+    build:
+      context: .
     container_name: jupyter
     depends_on:
       - spark-master
@@ -151,26 +152,8 @@ services:
       - "8888:8888"
     environment:
       JUPYTER_ENABLE_LAB: "yes"
-      SPARK_OPTS: >
-        --master=spark://spark-master:7077
-        --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
-        --conf spark.sql.catalog.iceberg=org.apache.iceberg.spark.SparkCatalog
-        --conf spark.sql.catalog.iceberg.type=nessie
-        --conf spark.sql.catalog.iceberg.uri=http://nessie:19120/api/v1
-        --conf spark.sql.catalog.iceberg.ref=main
-        --conf spark.sql.catalog.iceberg.catalog-impl=org.apache.iceberg.aws.s3.S3FileIO
-        --conf spark.sql.catalog.iceberg.s3.endpoint=http://minio:9000
-        --conf spark.sql.catalog.iceberg.s3.path-style-access=true
-        --conf spark.sql.catalog.iceberg.warehouse=s3a://nessie
-        --conf spark.hadoop.fs.s3a.access.key=minioadmin
-        --conf spark.hadoop.fs.s3a.secret.key=minioadmin
-        --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000
-        --conf spark.hadoop.fs.s3a.path.style.access=true
-        --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
     volumes:
       - jupyter_notebooks:/home/jovyan/work
-    command: >
-      start.sh jupyter lab --LabApp.token='' --LabApp.password=''
     networks:
       - app-network
 

docker-compose.yml renamed to nessie-stack/docker-compose.yml

@@ -0,0 +1,18 @@
+# Default Spark configuration
+spark.master                     spark://spark-master:7077
+spark.driver.memory              1g
+spark.executor.memory            1g
+spark.sql.extensions             org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.projectnessie.spark.extensions.NessieSparkSessionExtensions
+spark.sql.catalog.iceberg        org.apache.iceberg.spark.SparkCatalog
+spark.sql.catalog.iceberg.type   nessie
+spark.sql.catalog.iceberg.uri    http://nessie:19120/api/v1
+spark.sql.catalog.iceberg.ref    main
+spark.sql.catalog.iceberg.catalog-impl    org.apache.iceberg.aws.s3.S3FileIO
+spark.sql.catalog.iceberg.s3.endpoint     http://minio:9000
+spark.sql.catalog.iceberg.s3.path-style-access    true
+spark.sql.catalog.iceberg.warehouse       s3a://nessie
+spark.hadoop.fs.s3a.access.key   minioadmin
+spark.hadoop.fs.s3a.secret.key   minioadmin
+spark.hadoop.fs.s3a.endpoint     http://minio:9000
+spark.hadoop.fs.s3a.path.style.access     true
+spark.hadoop.fs.s3a.impl         org.apache.hadoop.fs.s3a.S3AFileSystem