chore: trying stuff

Author: reinthal

nessie-stack/jaffle-shop-classic-spark/.user.yml

@@ -0,0 +1 @@
+id: 22ed2a5a-1778-4431-ab2a-b546c4c3a1e4

nessie-stack/jaffle-shop-classic-spark/dbt_project.yml

@@ -3,7 +3,7 @@ name: 'jaffle_shop'
 config-version: 2
 version: '0.1'
 
-profile: 'jaffle_shop'
+profile: 'spark_profile'
 
 model-paths: ["models"]
 seed-paths: ["seeds"]
@@ -21,6 +21,7 @@ require-dbt-version: [">=1.0.0", "<2.0.0"]
 
 models:
   jaffle_shop:
+      +table_format: iceberg
       materialized: table
       staging:
         materialized: view

nessie-stack/jaffle-shop-classic-spark/profiles.yaml

@@ -0,0 +1,34 @@
+spark_profile:
+  target: dev
+  outputs:
+    dev:
+      type: spark
+      method: thrift
+      host: localhost
+      port: 10000 # Default Spark Thrift port, adjust if needed
+      user: # leave blank if not using auth
+      password: # leave blank if not using auth
+      database: default  # or your specific database
+      schema: default  # or your specific schema
+      threads: 1
+      connect_timeout: 60
+      connect_retries: 5
+      connect_retry_wait: 5
+      # Iceberg specific configurations
+      catalog: iceberg  # The name of your Iceberg catalog
+
+      # Additional Spark configurations for Iceberg
+      connect_args:
+        spark.jars.packages: org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.projectnessie.nessie-integrations:nessie-spark-extensions-3.5_2.12:0.102.5
+        spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.projectnessie.spark.extensions.NessieSparkSessionExtensions
+        spark.sql.catalog.iceberg: org.apache.iceberg.spark.SparkCatalog
+        spark.sql.catalog.iceberg.type: nessie
+        spark.sql.catalog.iceberg.warehouse: s3a://warehouse  
+        spark.sql.catalog.iceberg.uri: http://nessie:19120/api/v1
+        spark.sql.catalog.iceberg.ref: main
+        spark.sql.catalog.iceberg.io-impl: org.apache.iceberg.aws.s3.S3FileIO
+        spark.sql.catalog.iceberg.s3.endpoint: http://minio:9000
+        spark.sql.catalog.iceberg.s3.path-style-access: true
+        spark.sql.catalog.iceberg.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+        spark.sql.catalog.iceberg.s3.access.key: minioadmin
+        spark.sql.catalog.iceberg.s3.secret.key: minioadmin

nessie-stack/jaffle-shop-classic-spark/profiles.yml

@@ -21,7 +21,6 @@
 # Example:
 # Iceberg extensions
 spark.sql.extensions                              org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.projectnessie.spark.extensions.NessieSparkSessionExtensions
-
 # Iceberg catalog configuration
 spark.sql.catalog.iceberg                         org.apache.iceberg.spark.SparkCatalog
 spark.sql.catalog.iceberg.type                    nessie