Ele 4724 dbt fusion support

.github/workflows/test-all-warehouses.yml

@@ -37,7 +37,6 @@ jobs:
       matrix:
         dbt-version:
           ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) ||
-          ! contains(github.event_name, 'pull_request') && fromJSON('["1.8.0", "latest_official"]') ||
           fromJSON('["latest_official"]') }}
         warehouse-type:
           [
@@ -57,6 +56,14 @@ jobs:
             warehouse-type: postgres
           - dbt-version: "${{ inputs.dbt-version || 'latest_pre' }}"
             warehouse-type: postgres
+          - dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
+            warehouse-type: snowflake
+          - dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
+            warehouse-type: bigquery
+          - dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
+            warehouse-type: redshift
+          - dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
+            warehouse-type: databricks_catalog
     uses: ./.github/workflows/test-warehouse.yml
     with:
       warehouse-type: ${{ matrix.warehouse-type }}

.github/workflows/test-warehouse.yml

@@ -124,37 +124,44 @@ jobs:
         run: pip install databricks-sql-connector==2.9.3
 
       - name: Install dbt
+        if: ${{ inputs.dbt-version != 'fusion' }}
         run:
           pip install${{ (inputs.dbt-version == 'latest_pre' && ' --pre') || '' }}
           "dbt-core${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}"
           "dbt-${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || (inputs.warehouse-type == 'spark' && 'spark[PyHive]') || (inputs.warehouse-type == 'athena' && 'athena-community') || inputs.warehouse-type }}${{ (!startsWith(inputs.dbt-version, 'latest') && format('~={0}', inputs.dbt-version)) || '' }}"
 
+      - name: Install dbt-fusion
+        if: inputs.dbt-version == 'fusion'
+        run: |
+          curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s -- --update
+
       - name: Install Elementary
         run: pip install "./elementary[${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || inputs.warehouse-type }}]"
 
-      - name: Install dependencies
-        working-directory: ${{ env.TESTS_DIR }}
-        run: |
-          dbt deps --project-dir dbt_project
-          pip install -r requirements.txt
-
       - name: Write dbt profiles
         env:
-          PROFILES_YML: ${{ secrets.CI_PROFILES_YML }}
+          PROFILES_YML: ${{ (inputs.dbt-version == 'fusion' && secrets.CI_PROFILES_YML_FUSION) || secrets.CI_PROFILES_YML }}
         run: |
           mkdir -p ~/.dbt
-          DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
+          DBT_VERSION=$(echo "${{ inputs.dbt-version }}" | sed 's/\.//g')
           UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/[-\/]/_/g")
           echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/dbt_pkg_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml
 
+      - name: Install dependencies
+        working-directory: ${{ env.TESTS_DIR }}
+        run: |
+          ${{ (inputs.dbt-version == 'fusion' && '~/.local/bin/dbt') || 'dbt' }} deps --project-dir dbt_project
+          ln -sfn ${{ github.workspace }}/dbt-data-reliability dbt_project/dbt_packages/elementary
+          pip install -r requirements.txt
+
       - name: Check DWH connection
         working-directory: ${{ env.TESTS_DIR }}
         run: |
-          dbt debug -t "${{ inputs.warehouse-type }}"
+          ${{ (inputs.dbt-version == 'fusion' && '~/.local/bin/dbt') || 'dbt' }} debug -t "${{ inputs.warehouse-type }}"
 
       - name: Test
         working-directory: "${{ env.TESTS_DIR }}/tests"
-        run: py.test -n8 -vvv --target "${{ inputs.warehouse-type }}" --junit-xml=test-results.xml --html=detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html --self-contained-html --clear-on-end
+        run: py.test -n8 -vvv --target "${{ inputs.warehouse-type }}" --junit-xml=test-results.xml --html=detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html --self-contained-html --clear-on-end ${{ (inputs.dbt-version == 'fusion' && '--runner-method fusion') || '' }}
 
       - name: Upload test results
         if: always()

.gitignore

@@ -25,3 +25,6 @@ __pycache__/
 
 # vscode
 .vscode/
+dbt_internal_packages/
+
+/package-lock.yml

integration_tests/dbt_project/.gitignore

@@ -1,2 +1,3 @@
 data
 models/tmp
+dbt_internal_packages/

integration_tests/dbt_project/dbt_project.yml

@@ -18,7 +18,6 @@ clean-targets: # directories to be removed by `dbt clean`
 
 vars:
   debug_logs: "{{ env_var('DBT_EDR_DEBUG', False) }}"
-  mute_ensure_materialization_override: true
 
 models:
   elementary_tests:

integration_tests/dbt_project/macros/create_all_types_table.sql

@@ -31,9 +31,7 @@
         CURRENT_TIME() as time_col,
         CURRENT_TIMESTAMP() as timestamp_col,
     {% endset %}
-    {% set create_table_query = dbt.create_table_as(false, relation, sql_query) %}
-    {% do elementary.edr_log(create_table_query) %}
-    {% do elementary.run_query(create_table_query) %}
+    {% do elementary.edr_create_table_as(false, relation, sql_query) %}
 {% endmacro %}
 
 {% macro snowflake__create_all_types_table() %}
@@ -81,9 +79,7 @@
         [1,2,3] as array_col,
         TO_GEOGRAPHY('POINT(-122.35 37.55)') as geography_col
     {% endset %}
-    {% set create_table_query = dbt.create_table_as(false, relation, sql_query) %}
-    {% do elementary.edr_log(create_table_query) %}
-    {% do elementary.run_query(create_table_query) %}
+    {% do elementary.edr_create_table_as(false, relation, sql_query) %}
 {% endmacro %}
 
 {% macro redshift__create_all_types_table() %}
@@ -123,9 +119,7 @@
         ST_GeogFromText('SRID=4324;POLYGON((0 0,0 1,1 1,10 10,1 0,0 0))') as geography_col,
         JSON_PARSE('{"data_type": "super"}') as super_col
     {% endset %}
-    {% set create_table_query = dbt.create_table_as(false, relation, sql_query) %}
-    {% do elementary.edr_log(create_table_query) %}
-    {% do elementary.run_query(create_table_query) %}
+    {% do elementary.edr_create_table_as(false, relation, sql_query) %}
 
 {% endmacro %}
 
@@ -184,9 +178,7 @@
         'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid as uuid_col,
         xmlcomment('text') as xml_col
     {% endset %}
-    {% set create_table_query = dbt.create_table_as(false, relation, sql_query) %}
-    {% do elementary.edr_log(create_table_query) %}
-    {% do elementary.run_query(create_table_query) %}
+    {% do elementary.edr_create_table_as(false, relation, sql_query) %}
 {% endmacro %}
 
 {% macro default__create_all_types_table() %}

integration_tests/dbt_project/macros/materializations.sql

@@ -1,19 +1,19 @@
 {% materialization test, default %}
   {% if var('enable_elementary_test_materialization', false) %}
-    {% do return(elementary.materialization_test_default.call_macro()) %}
+    {% do return(elementary.materialization_test_default()) %}
   {% else %}
-    {% do return(dbt.materialization_test_default.call_macro()) %}
+    {% do return(dbt.materialization_test_default()) %}
   {% endif %}
 {% endmaterialization %}
 
 {% materialization test, adapter="snowflake" %}
   {% if var('enable_elementary_test_materialization', false) %}
-    {% do return(elementary.materialization_test_snowflake.call_macro()) %}
+    {% do return(elementary.materialization_test_snowflake()) %}
   {% else %}
     {% if dbt.materialization_test_snowflake %}
-      {% do return(dbt.materialization_test_snowflake.call_macro()) %}
+      {% do return(dbt.materialization_test_snowflake()) %}
     {% else %}
-      {% do return(dbt.materialization_test_default.call_macro()) %}
+      {% do return(dbt.materialization_test_default()) %}
     {% endif %}
   {% endif %}
 {% endmaterialization %}

integration_tests/dbt_project/macros/replace_empty_strings_with_nulls.sql

@@ -0,0 +1,18 @@
+{% macro replace_empty_strings_with_nulls(table_name) %}
+    {% set relation = ref(table_name) %}
+    {% set columns = adapter.get_columns_in_relation(relation) %}
+
+    {% for col in columns %}
+        {% set data_type = elementary.get_column_data_type(col) %}
+        {% set normalized_data_type = elementary.normalize_data_type(data_type) %}
+
+        {% if normalized_data_type == "string" %}
+            {% set update_query %}
+                update {{ relation }}
+                set {{ col["name"] }} = NULL
+                where {{ col["name"] }} = ''
+            {% endset %}
+            {% do elementary.run_query(update_query) %}
+        {% endif %}
+    {% endfor %}
+{% endmacro %}

integration_tests/dbt_project/models/exposures.yml

@@ -15,11 +15,12 @@ exposures:
     owner:
       name: Callum McData
       email: [email protected]
-    meta:
-      referenced_columns:
-        - column_name: id
-          data_type: numeric
-          node: ref('customers')
+    config:
+      meta:
+        referenced_columns:
+          - column_name: id
+            data_type: numeric
+            node: ref('customers')
 
   - name: orders
     label: Returned Orders
@@ -35,8 +36,9 @@ exposures:
     owner:
       name: Callum McData
       email: [email protected]
-    meta:
-      referenced_columns:
-        - column_name: "order_id"
-          data_type: "string"
-        - column_name: "ZOMG"
+    config:
+      meta:
+        referenced_columns:
+          - column_name: "order_id"
+            data_type: "string"
+          - column_name: "ZOMG"

integration_tests/dbt_project/models/schema.yml

@@ -5,7 +5,8 @@ models:
     description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
     tests:
       - elementary.exposure_schema_validity:
-          tags: [exposure_customers]
+          config:
+            tags: [exposure_customers]
 
     columns:
       - name: id
@@ -20,7 +21,8 @@ models:
 
     tests:
       - elementary.exposure_schema_validity:
-          tags: [exposure_orders]
+          config:
+            tags: [exposure_orders]
 
     columns:
       - name: order_id

integration_tests/dbt_project/packages.yml

@@ -1,10 +1,4 @@
 packages:
   - local: ../../
   - package: dbt-labs/dbt_utils
-    version: |
-      {%- set minor_to_utils_range_map = {
-        "0": [">=0.8.0", "<0.9.0"],
-        "1": [">=0.8.0", "<0.9.0"],
-        "2": [">=0.8.0", "<1.0.0"],
-      } -%}
-      {{- minor_to_utils_range_map.get(dbt_version.split('.')[1], [">=0.8.0", "<2.0.0"]) -}}
+    version: [">=0.8.0", "<2.0.0"]

integration_tests/deprecated_tests/dbt_project.yml

@@ -20,7 +20,6 @@ vars:
   days_back: 30
   debug_logs: "{{ env_var('DBT_EDR_DEBUG', False) }}"
   custom_run_started_at: "{{ modules.datetime.datetime.utcfromtimestamp(0) }}"
-  mute_ensure_materialization_override: true
 
 seeds:
   +schema: test_seeds

integration_tests/deprecated_tests/macros/system/materializations.sql

@@ -1,7 +1,7 @@
 {% materialization test, default %}
-  {% do return(elementary.materialization_test_default.call_macro()) %}
+  {% do return(elementary.materialization_test_default()) %}
 {% endmaterialization %}
 
 {% materialization test, adapter="snowflake" %}
-  {% do return(elementary.materialization_test_snowflake.call_macro()) %}
+  {% do return(elementary.materialization_test_snowflake()) %}
 {% endmaterialization %}