74 tests pass

Author: Michael Myaskovsky

integration_tests/dbt_project/macros/get_anomaly_config.sql

@@ -1,4 +1,8 @@
 {% macro get_anomaly_config(model_config, config) %}
+    {{ return(adapter.dispatch('get_anomaly_config', 'elementary')(model_config, config)) }}
+{% endmacro %}
+
+{% macro default__get_anomaly_config(model_config, config) %}
   {% set mock_model = {
     "alias": "mock_model",
     "config": {
@@ -19,4 +23,27 @@
     }
   }) %}
   {% do return(elementary.get_anomalies_test_configuration(api.Relation.create("db", "schema", "mock_model"), **config)[0]) %}
+{% endmacro %}
+
+{% macro clickhouse__get_anomaly_config(model_config, config) %}
+  {% set mock_model = {
+    "alias": "mock_model",
+    "config": {
+      "elementary": model_config
+    }
+  } %}
+  {# trick elementary into thinking this is the running model #}
+  {% do context.update({
+    "model": {
+      "depends_on": {
+        "nodes": ["id"]
+      }
+    },
+    "graph": {
+      "nodes": {
+        "id": mock_model
+      }
+    }
+  }) %}
+  {% do return(elementary.get_anomalies_test_configuration(api.Relation.create("schema", "schema", "mock_model"), **config)[0]) %}
 {% endmacro %}

integration_tests/tests/test_dbt_artifacts/test_artifacts.py

@@ -98,7 +98,7 @@ def test_source_freshness_results(test_id: str, dbt_project: DbtProject):
         "sources": [
             {
                 "name": "test_source",
-                "database": "{{target.database}}",
+                "database": "{{target.database if target.type != 'clickhouse' else target.schema}}",
                 "schema": "{{target.schema}}",
                 "tables": [
                     {

macros/edr/data_monitoring/monitors_query/table_monitoring_query.sql

@@ -227,7 +227,7 @@
     consecutive_updates_freshness as (
         select
             timestamp_val as update_timestamp,
-            {{ elementary.timediff('second', 'lag(timestamp_val) over (order by timestamp_val)', 'timestamp_val') }} as freshness
+            {{ elementary.timediff('second', elementary.lag('timestamp_val') ~ ' over (order by timestamp_val)', 'timestamp_val') }} as freshness
         from unique_timestamps
     ),
     time_filtered_consecutive_updates_freshness as (
@@ -300,9 +300,21 @@
 {% endmacro %}
 
 {% macro get_no_timestamp_event_freshness_query(metric, metric_properties) %}
+    {{ return(adapter.dispatch('get_no_timestamp_event_freshness_query', 'elementary')(metric, metric_properties)) }}
+{% endmacro %}
+
+{% macro default__get_no_timestamp_event_freshness_query(metric, metric_properties) %}
     select
         {{ elementary.const_as_string(metric.name) }} as metric_name,
         {{ elementary.const_as_string("event_freshness") }} as metric_type,
         {{ elementary.timediff('second', elementary.edr_cast_as_timestamp("max({})".format(metric_properties.event_timestamp_column)), elementary.edr_quote(elementary.get_run_started_at())) }} as metric_value
     from monitored_table
 {% endmacro %}
+
+{% macro clickhouse__get_no_timestamp_event_freshness_query(metric, metric_properties) %}
+    select
+        {{ elementary.const_as_string(metric.name) }} as metric_name,
+        {{ elementary.const_as_string("event_freshness") }} as metric_type,
+        {{ elementary.timediff('second', elementary.edr_cast_as_timestamp("max(toDateTime64({}, 3))".format(metric_properties.event_timestamp_column)), elementary.edr_quote(elementary.get_run_started_at())) }} as metric_value
+    from monitored_table
+{% endmacro %}

macros/edr/system/system_utils/full_names.sql

@@ -33,6 +33,20 @@
 {% endmacro %}
 
 
+{% macro clickhouse__full_name_split(part_name) %}
+    {%- if part_name == 'database_name' -%}
+        {%- set part_index = 1 -%}
+    {%- elif part_name == 'schema_name' -%}
+        {%- set part_index = 2 -%}
+    {%- elif part_name == 'table_name' -%}
+        {%- set part_index = 3 -%}
+    {%- else -%}
+        {{ return('') }}
+    {%- endif -%}
+    trim(BOTH '"' FROM splitByChar('.', full_table_name)[{{ part_index }}]) AS {{ part_name }}
+{% endmacro %}
+
+
 {% macro bigquery__full_name_split(part_name) %}
     {%- if part_name == 'database_name' -%}
         {%- set part_index = 0 %}
@@ -88,6 +102,7 @@
     {{ return(full_table_name) }}
 {% endmacro %}
 
+
 {% macro model_node_to_full_name(model_node) %}
     {% set identifier = model_node.identifier or model_node.alias %}
     {%- if model_node.database %}

macros/edr/tests/test_column_anomalies.sql

@@ -42,7 +42,6 @@
             {{ exceptions.raise_compiler_error("Failed to create test configuration dict for test `{}`".format(test_table_name)) }}
         {%- endif %}
         {{ elementary.debug_log('test configuration - ' ~ test_configuration) }}
-
         {%- set column_obj_and_monitors = elementary.get_column_obj_and_monitors(model_relation, column_name, column_anomalies) -%}
         {%- if not column_obj_and_monitors -%}
             {{ exceptions.raise_compiler_error("Unable to find column `{}` in `{}`".format(column_name, full_table_name)) }}

macros/edr/tests/test_schema_changes_from_baseline.sql

@@ -5,7 +5,6 @@
         {% if not model_relation %}
             {{ exceptions.raise_compiler_error("Unsupported model: " ~ model ~ " (this might happen if you override 'ref' or 'source)") }}
         {% endif %}
-
         {%- if elementary.is_ephemeral_model(model_relation) %}
             {{ exceptions.raise_compiler_error("The test is not supported for ephemeral models, model name: {}".format(model_relation.identifier)) }}
         {%- endif %}

macros/utils/cross_db_utils/lag.sql

@@ -0,0 +1,11 @@
+{% macro lag(column, offset=1) %}
+    {{ return(adapter.dispatch('lag', 'elementary')(column, offset)) }}
+{% endmacro %}
+
+{% macro default__lag(column, offset=1) %}
+    lag({{ column }}, {{ offset }})
+{% endmacro %}
+
+{% macro clickhouse__lag(column, offset=1) %}
+    lagInFrame({{ column }}, {{ offset }})
+{% endmacro %}

macros/utils/data_types/data_type_list.sql

@@ -131,3 +131,23 @@
     {%- endif %}
 
 {% endmacro %}
+
+{% macro clickhouse__data_type_list(data_type) %}
+    {% set string_list = ['String', 'FixedString', 'LowCardinality(String)'] | list %}
+    {% set numeric_list = ['Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64', 'Decimal', 'Decimal32', 'Decimal64', 'Decimal128'] | list %}
+    {% set timestamp_list = ['DateTime', 'DateTime64', 'Date', 'Date32'] | list %}
+    {% set boolean_list = ["UInt8","Bool"] | list %}
+
+    {%- if data_type == 'string' %}
+        {{ return(string_list) }}
+    {%- elif data_type == 'numeric' %}
+        {{ return(numeric_list) }}
+    {%- elif data_type == 'timestamp' %}
+        {{ return(timestamp_list) }}
+    {%- elif data_type == "boolean" %}
+        {{ return(boolean_list) }}
+    {%- else %}
+        {{ return([]) }}
+    {%- endif %}
+
+{% endmacro %}