diff --git a/.github/workflows/dbt_run.yml b/.github/workflows/dbt_run.yml index 4b18987..b4fd663 100644 --- a/.github/workflows/dbt_run.yml +++ b/.github/workflows/dbt_run.yml @@ -26,6 +26,10 @@ jobs: - name: Install dependencies with uv run: uv sync + - name: Setup variables + run: | + echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV + - name: Run dbt deps run: uv run dbt deps @@ -33,16 +37,16 @@ jobs: run: "./scripts/activate-trino-cluster.sh" - name: Run dbt compile - run: uv run dbt compile + run: uv run dbt compile $PROFILE - name: Run dbt run (full refresh) - run: uv run dbt run --full-refresh + run: uv run dbt run $PROFILE --full-refresh - name: Run dbt test (after full refresh) - run: uv run dbt test + run: uv run dbt test $PROFILE - name: Run dbt run (incremental update) - run: uv run dbt run + run: uv run dbt run $PROFILE - name: Run dbt test (after incremental) - run: uv run dbt test \ No newline at end of file + run: uv run dbt test $PROFILE \ No newline at end of file diff --git a/macros/dune_dbt_overrides/get_custom_schema.sql b/macros/dune_dbt_overrides/get_custom_schema.sql index 48edc6e..5458d12 100644 --- a/macros/dune_dbt_overrides/get_custom_schema.sql +++ b/macros/dune_dbt_overrides/get_custom_schema.sql @@ -1,5 +1,5 @@ {# - goal: depending on how we handle dev vs. prod environments (i.e. s3 bucket names), we may not need this macro override + goal: depending on how we handle dev vs. prod environments (i.e. s3), we may not need this macro override Custom schema naming logic for dev vs prod environments @@ -19,19 +19,22 @@ {% macro generate_schema_name(custom_schema_name, node) -%} {%- set default_schema = target.schema -%} - {%- if custom_schema_name is none -%} - - {{ default_schema }} + {%- if target.name == 'prod' and custom_schema_name is not none -%} + {# prod environment #} + {{ custom_schema_name | trim }} - {%- elif target.name == 'prod' -%} + {%- elif target.schema.startswith("github_actions") -%} + {# test environment, CI pipeline #} + {{ 'test_schema' }} - {{ custom_schema_name | trim }} + {%- elif custom_schema_name is none -%} + + {{ default_schema }} {%- else -%} - + {# dev environment, running locally #} {{ default_schema }}_{{ custom_schema_name | trim }} {%- endif -%} -{%- endmacro %} - +{%- endmacro %} \ No newline at end of file diff --git a/models/_schema.yml b/models/_schema.yml index ee4ba76..5009e94 100644 --- a/models/_schema.yml +++ b/models/_schema.yml @@ -6,5 +6,30 @@ models: columns: - name: block_number description: "The unique block number in last 1 day" + - name: block_date + description: "The date of the block" + - name: total_tx_per_block + description: "The total number of transactions per block" + - name: dbt_template_table_model + description: "A starter dbt table model" + columns: + - name: block_number + description: "The unique block number in last 1 day" + - name: block_date + description: "The date of the block" + - name: total_tx_per_block + description: "The total number of transactions per block" + - name: dbt_template_incremental_model + description: "A starter dbt incremental model" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_number + - block_date + columns: + - name: block_number + description: "The unique block number in last 1 day" + - name: block_date + description: "The date of the block" - name: total_tx_per_block description: "The total number of transactions per block" \ No newline at end of file diff --git a/models/dbt_template_incremental_model.sql b/models/dbt_template_incremental_model.sql index 56f2658..38f950c 100644 --- a/models/dbt_template_incremental_model.sql +++ b/models/dbt_template_incremental_model.sql @@ -5,8 +5,8 @@ #} {{ config( - schema = 'dbt_template' - , alias = 'incremental_model' + schema = 'test_schema' + , alias = 'dbt_template_incremental_model' , materialized = 'incremental' , incremental_strategy = 'merge' , unique_key = ['block_number', 'block_date'] @@ -18,7 +18,7 @@ select block_number , block_date - , count(1) as total_tx_per_block + , count(1) as total_tx_per_block -- count per block from {{ source('ethereum', 'transactions') }} where diff --git a/models/dbt_template_table_model.sql b/models/dbt_template_table_model.sql index 0ea8451..fd97229 100644 --- a/models/dbt_template_table_model.sql +++ b/models/dbt_template_table_model.sql @@ -7,8 +7,8 @@ #} {{ config( - schema = 'dbt_template' - , alias = 'table_model' + schema = 'test_schema' + , alias = 'dbt_template_table_model' , materialized = 'table' , on_table_exists = 'replace' ) @@ -16,10 +16,12 @@ select block_number + , block_date , count(1) as total_tx_per_block from {{ source('ethereum', 'transactions') }} where block_date >= now() - interval '1' day group by - block_number \ No newline at end of file + block_number + , block_date \ No newline at end of file diff --git a/models/dbt_template_view_model.sql b/models/dbt_template_view_model.sql index 24a0aef..bc1e337 100644 --- a/models/dbt_template_view_model.sql +++ b/models/dbt_template_view_model.sql @@ -1,16 +1,18 @@ {{ config( - schema = 'dbt_template' - , alias = 'view_model' + schema = 'test_schema' + , alias = 'dbt_template_view_model' , materialized = 'view' ) }} select block_number + , block_date , count(1) as total_tx_per_block from {{ source('ethereum', 'transactions') }} where block_date >= now() - interval '1' day group by - block_number \ No newline at end of file + block_number + , block_date \ No newline at end of file diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..1b68932 --- /dev/null +++ b/packages.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<2.0.0"] + diff --git a/scripts/activate-trino-cluster.sh b/scripts/activate-trino-cluster.sh index 21d397e..e4e17b0 100755 --- a/scripts/activate-trino-cluster.sh +++ b/scripts/activate-trino-cluster.sh @@ -9,7 +9,7 @@ WAIT_TIME=15 until [ "$RETRY_COUNT" -ge "$MAX_RETRIES" ] do - uv run dbt debug && break + uv run dbt debug $PROFILE && break RETRY_COUNT=$((RETRY_COUNT+1)) if [ "$RETRY_COUNT" -ge "$MAX_RETRIES" ]; then echo "Max retries reached, failing..."