Merge pull request #1 from duneanalytics/test-ci

fix CI pipeline, update model naming standards, import dbt_utils

Author: jeff-dude

.github/workflows/dbt_run.yml

@@ -26,23 +26,27 @@ jobs:
       - name: Install dependencies with uv
         run: uv sync
 
+      - name: Setup variables
+        run: |
+          echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
+
       - name: Run dbt deps
         run: uv run dbt deps
 
       - name: Activate Trino cluster
         run: "./scripts/activate-trino-cluster.sh"
 
       - name: Run dbt compile
-        run: uv run dbt compile
+        run: uv run dbt compile $PROFILE
 
       - name: Run dbt run (full refresh)
-        run: uv run dbt run --full-refresh
+        run: uv run dbt run $PROFILE --full-refresh
 
       - name: Run dbt test (after full refresh)
-        run: uv run dbt test
+        run: uv run dbt test $PROFILE
 
       - name: Run dbt run (incremental update)
-        run: uv run dbt run
+        run: uv run dbt run $PROFILE
 
       - name: Run dbt test (after incremental)
-        run: uv run dbt test
+        run: uv run dbt test $PROFILE

macros/dune_dbt_overrides/get_custom_schema.sql

@@ -1,5 +1,5 @@
 {#
-    goal: depending on how we handle dev vs. prod environments (i.e. s3 bucket names), we may not need this macro override
+    goal: depending on how we handle dev vs. prod environments (i.e. s3), we may not need this macro override
 
     Custom schema naming logic for dev vs prod environments
 
@@ -19,19 +19,22 @@
 {% macro generate_schema_name(custom_schema_name, node) -%}
 
     {%- set default_schema = target.schema -%}
-    {%- if custom_schema_name is none -%}
-
-        {{ default_schema }}
+    {%- if target.name == 'prod' and custom_schema_name is not none -%}
+        {# prod environment #}
+        {{ custom_schema_name | trim }}
 
-    {%- elif target.name == 'prod' -%}
+    {%- elif target.schema.startswith("github_actions") -%}
+        {# test environment, CI pipeline #}
+        {{ 'test_schema' }}
 
-        {{ custom_schema_name | trim }}
+    {%- elif custom_schema_name is none -%}
+        
+        {{ default_schema }}
 
     {%- else -%}
-
+        {# dev environment, running locally #}
         {{ default_schema }}_{{ custom_schema_name | trim }}
 
     {%- endif -%}
 
-{%- endmacro %}
-
+{%- endmacro %}

models/_schema.yml

@@ -6,5 +6,30 @@ models:
     columns:
       - name: block_number
         description: "The unique block number in last 1 day"
+      - name: block_date
+        description: "The date of the block"
+      - name: total_tx_per_block
+        description: "The total number of transactions per block"
+  - name: dbt_template_table_model
+    description: "A starter dbt table model"
+    columns:
+      - name: block_number
+        description: "The unique block number in last 1 day"
+      - name: block_date
+        description: "The date of the block"
+      - name: total_tx_per_block
+        description: "The total number of transactions per block"
+  - name: dbt_template_incremental_model
+    description: "A starter dbt incremental model"
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - block_number
+            - block_date
+    columns:
+      - name: block_number
+        description: "The unique block number in last 1 day"
+      - name: block_date
+        description: "The date of the block"
       - name: total_tx_per_block
         description: "The total number of transactions per block"

models/dbt_template_incremental_model.sql

@@ -5,8 +5,8 @@
 #}
 
 {{ config(
-    schema = 'dbt_template'
-    , alias = 'incremental_model'
+    schema = 'test_schema'
+    , alias = 'dbt_template_incremental_model'
     , materialized = 'incremental'
     , incremental_strategy = 'merge'
     , unique_key = ['block_number', 'block_date']
@@ -18,7 +18,7 @@
 select
     block_number
     , block_date
-    , count(1) as total_tx_per_block
+    , count(1) as total_tx_per_block -- count per block
 from
     {{ source('ethereum', 'transactions') }}
 where

models/dbt_template_table_model.sql

@@ -7,19 +7,21 @@
 #}
 
 {{ config(
-    schema = 'dbt_template'
-    , alias = 'table_model'
+    schema = 'test_schema'
+    , alias = 'dbt_template_table_model'
     , materialized = 'table'
     , on_table_exists = 'replace'
 )
 }}
 
 select
     block_number
+    , block_date
     , count(1) as total_tx_per_block
 from
     {{ source('ethereum', 'transactions') }}
 where
     block_date >= now() - interval '1' day
 group by
-    block_number
+    block_number
+    , block_date

models/dbt_template_view_model.sql

@@ -1,16 +1,18 @@
 {{ config(
-    schema = 'dbt_template'
-    , alias = 'view_model'
+    schema = 'test_schema'
+    , alias = 'dbt_template_view_model'
     , materialized = 'view'
 )
 }}
 
 select
     block_number
+    , block_date
     , count(1) as total_tx_per_block
 from
     {{ source('ethereum', 'transactions') }}
 where
     block_date >= now() - interval '1' day
 group by
-    block_number
+    block_number
+    , block_date

packages.yml

@@ -0,0 +1,4 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: [">=1.0.0", "<2.0.0"]
+

scripts/activate-trino-cluster.sh

@@ -9,7 +9,7 @@ WAIT_TIME=15
 
 until [ "$RETRY_COUNT" -ge "$MAX_RETRIES" ]
 do
-  uv run dbt debug && break
+  uv run dbt debug $PROFILE && break
   RETRY_COUNT=$((RETRY_COUNT+1))
   if [ "$RETRY_COUNT" -ge "$MAX_RETRIES" ]; then
     echo "Max retries reached, failing..."