solved iceberg ez

Author: reinthal

nessie-stack/jaffle-shop-classic-spark/models/employees2.sql

@@ -0,0 +1,10 @@
+{{
+  config(
+    materialized='table',
+    file_format='iceberg'
+  )
+}}
+
+-- Simple SELECT to verify that dbt can read from your manual table
+-- and write to a new Iceberg table
+SELECT * FROM {{ source('test', 'default.employees') }}

nessie-stack/jaffle-shop-classic-spark/models/sources.yml

@@ -0,0 +1,9 @@
+version: 2
+
+sources:
+  - name: test
+    database: nessie
+    schema: nessie
+    tables:
+      - name: default.employees
+        description: "Test table"

nessie-stack/jaffle-shop-classic-spark/profiles.yml

@@ -5,11 +5,10 @@ spark_profile:
       type: spark
       method: thrift
       host: localhost
-      port: 10001 # Default Spark Thrift port, adjust if needed
+      port: 10000 # Default Spark Thrift port, adjust if needed
       user: # leave blank if not using auth
       password: # leave blank if not using auth
-      database: default  # or your specific database
-      schema: default  # or your specific schema
+      schema: nessie.default  # or your specific schema
       threads: 1
       connect_timeout: 60
       connect_retries: 5
@@ -21,21 +20,18 @@ spark_profile:
       connect_args:
         spark.jars.packages: org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.projectnessie.nessie-integrations:nessie-spark-extensions-3.5_2.12:0.102.5,org.apache.hadoop:hadoop-aws:3.3.4
         spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.projectnessie.spark.extensions.NessieSparkSessionExtensions
-        spark.sql.catalog.iceberg: org.apache.iceberg.spark.SparkCatalog
-        spark.sql.catalog.iceberg.type: nessie
-        spark.sql.catalog.iceberg.warehouse: s3a://warehouse
-        spark.sql.catalog.iceberg.uri: http://nessie:19120/api/v1
-        spark.sql.catalog.iceberg.ref: main
-        spark.sql.catalog.iceberg.io-impl: org.apache.iceberg.aws.s3.S3FileIO
-        spark.sql.catalog.iceberg.s3.endpoint: http://minio:9000
-        spark.sql.catalog.iceberg.s3.path-style-access: true
-        spark.sql.catalog.iceberg.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
-        spark.sql.catalog.iceberg.s3.access.key: minioadmin
-        spark.sql.catalog.iceberg.s3.secret.key: minioadmin
+        spark.sql.catalog.nessie: org.apache.iceberg.spark.SparkCatalog
+        spark.sql.catalog.nessie.type: nessie
+        spark.sql.catalog.nessie.warehouse: s3a://warehouse
+        spark.sql.catalog.nessie.uri: http://nessie:19120/api/v1
+        spark.sql.catalog.nessie.ref: main
+        spark.sql.catalog.nessie.io-impl: org.apache.iceberg.aws.s3.S3FileIO
+        spark.sql.catalog.nessie.s3.endpoint: http://minio:9000
+        spark.sql.catalog.nessie.s3.path-style-access: true
+        spark.sql.catalog.nessie.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+        spark.sql.catalog.nessie.s3.access.key: minioadmin
+        spark.sql.catalog.nessie.s3.secret.key: minioadmin
         # Add these new configurations
-        spark.sql.defaultCatalog: iceberg
-        spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
-        spark.sql.catalog.spark_catalog.type: nessie
         spark.hadoop.fs.s3a.access.key: minioadmin
         spark.hadoop.fs.s3a.secret.key: minioadmin
         spark.hadoop.fs.s3a.endpoint: http://minio:9000