Merge pull request #375 from elementary-data/ele-1061-databricks-enabledfalse-gets-override-by-user-config

Made databricks avoid running info schema models even if user `enabled: true`

Author: elongl

macros/edr/metadata_collection/get_columns_by_schemas.sql

@@ -2,6 +2,6 @@
     {%- if configured_schemas | length > 0 -%}
         {{ elementary.union_macro_queries(configured_schemas, elementary.get_columns_from_information_schema) }}
     {%- else %}
-        {{ elementary.empty_table([('full_table_name', 'string'), ('database_name', 'string'), ('schema_name', 'string'), ('table_name', 'string'), ('column_name', 'string'), ('data_type', 'string')]) }}
+        {{ elementary.get_empty_columns_from_information_schema_table() }}
     {%- endif %}
 {% endmacro %}

macros/edr/metadata_collection/get_columns_from_information_schema.sql

@@ -43,4 +43,12 @@
     from information_schema.columns
     where upper(table_schema) = upper('{{ schema_name }}')
 
-{% endmacro %}
+{% endmacro %}
+
+{% macro spark__get_columns_from_information_schema(database_name, schema_name) %}
+    {{ elementary.get_empty_columns_from_information_schema_table() }}
+{% endmacro %}
+
+{% macro get_empty_columns_from_information_schema_table() %}
+    {{ elementary.empty_table([('full_table_name', 'string'), ('database_name', 'string'), ('schema_name', 'string'), ('table_name', 'string'), ('column_name', 'string'), ('data_type', 'string')]) }}
+{% endmacro %}

macros/edr/system/system_utils/get_config_var.sql

@@ -32,7 +32,6 @@
     'disable_tests_results': false,
     'disable_dbt_artifacts_autoupload': false,
     'disable_dbt_invocation_autoupload': false,
-    'enable_dbt_columns': true,
     'disable_skipped_model_alerts': true,
     'disable_skipped_test_alerts': true,
     'dbt_artifacts_chunk_size': 5000,

models/edr/alerts/alerts_schema_changes.sql

@@ -2,7 +2,6 @@
   config(
     materialized = 'view',
     bind=False,
-    enabled = target.type != 'databricks' and target.type != 'spark' | as_bool()
   )
 }}
 

models/edr/data_monitoring/schema_changes/schema_columns_snapshot.sql

@@ -2,7 +2,6 @@
   config(
     materialized='incremental',
     unique_key = 'column_state_id',
-    enabled = target.type != 'databricks' and target.type != 'spark' | as_bool(),
     on_schema_change = 'append_new_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     meta={

models/edr/dbt_artifacts/dbt_columns.sql

@@ -1,7 +1,6 @@
 {{
   config(
     materialized = 'view' if not var('sync', false) else 'table',
-    enabled = elementary.get_config_var('enable_dbt_columns') and target.type != 'databricks' and target.type != 'spark' | as_bool()
   )
 }}
 

models/edr/metadata_store/filtered_information_schema_columns.sql

@@ -1,7 +1,6 @@
 {{
   config(
     materialized = 'view',
-    enabled = target.type != 'databricks' and target.type != 'spark' | as_bool()
   )
 }}
 
@@ -13,4 +12,4 @@ with filtered_information_schema_columns as (
 
 select *
 from filtered_information_schema_columns
-where full_table_name is not null
+where full_table_name is not null

models/edr/metadata_store/filtered_information_schema_tables.sql

@@ -35,9 +35,3 @@ models:
       Queries the columns view from the information schema of the schemas in the project.
       This view is generated using an adapter specific macro, as information schema is different between platforms.
       This is a view to make the work with the information schema more convenient.
-
-  - name: filtered_information_schema_tables
-    description: >
-      Queries the tables and schemas views from the information schema of the schemas in the project.
-      This view is generated using an adapter specific macro, as information schema is different between platforms.
-      This is a view to make the work with the information schema more convenient.