fixed a few more

Author: Michael Myaskovsky

macros/edr/data_monitoring/monitors/column_numeric_monitors.sql

@@ -31,13 +31,21 @@
 {%- endmacro %}
 
 {% macro clickhouse__standard_deviation(column_name) -%}
-    stddevPop(cast({{ column_name }} as {{ elementary.edr_type_float() }}))
+    stddevSamp(cast({{ column_name }} as {{ elementary.edr_type_float() }}))
 {%- endmacro %}
 
 {% macro variance(column_name) -%}
+    {{ return(adapter.dispatch('variance', 'elementary')(column_name)) }}
+{%- endmacro %}
+
+{% macro default__variance(column_name) -%}
     variance(cast({{ column_name }} as {{ elementary.edr_type_float() }}))
 {%- endmacro %}
 
+{% macro clickhouse__variance(column_name) -%}
+    varSamp(cast({{ column_name }} as {{ elementary.edr_type_float() }}))
+{%- endmacro %}
+
 {% macro sum(column_name) -%}
     sum(cast({{ column_name }} as {{ elementary.edr_type_float() }}))
 {%- endmacro %}

macros/edr/tests/test_utils/get_anomaly_query.sql

@@ -60,37 +60,24 @@ case when
         from anomaly_scores
       ),
 
-      {{ elementary.get_final_results_query(test_configuration) }}
-
-      select * from final_results
-      where {{ test_configuration.exclude_final_results }}
-    {%- endset -%}
-    {{- return(anomaly_query) -}}
-{% endmacro %}
-
-{% macro get_final_results_query(test_configuration) %}
-    {{ return(adapter.dispatch('get_final_results_query', 'elementary')(test_configuration)) }}
-{% endmacro %}
-
-{% macro default__get_final_results_query(test_configuration) %}
-    final_results as (
+      final_results as (
         select
             metric_value as value,
             training_avg as average,
             {# when there is an anomaly we would want to use the last value of the metric (lag), otherwise visually the expectations would look out of bounds #}
             case
                 when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' = 'spike' then
-                    lag(metric_value) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
+                    {{ elementary.lag('metric_value') }} over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
                 when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' != 'spike' then
-                    lag(min_metric_value) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
+                    {{ elementary.lag('min_metric_value') }} over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
                 when '{{ test_configuration.anomaly_direction }}' = 'spike' then metric_value
                 else min_metric_value 
             end as min_value,
             case
                 when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' = 'drop' then
-                    lag(metric_value) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
+                    {{ elementary.lag('metric_value') }} over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
                 when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' != 'drop' then
-                    lag(max_metric_value) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
+                    {{ elementary.lag('max_metric_value') }} over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
                 when '{{ test_configuration.anomaly_direction }}' = 'drop' then metric_value
                 else max_metric_value 
             end as max_value,
@@ -100,36 +87,11 @@ case when
         from anomaly_scores_with_is_anomalous
         order by bucket_end, dimension_value
     )
-{% endmacro %}
 
-{% macro clickhouse__get_final_results_query(test_configuration) %}
-    final_results as (
-        select
-            metric_value as value,
-            training_avg as average,
-            {# when there is an anomaly we would want to use the last value of the metric (lag), otherwise visually the expectations would look out of bounds #}
-            case
-                when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' = 'spike' then
-                    lagInFrame(metric_value, 1) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
-                when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' != 'spike' then
-                    lagInFrame(min_metric_value, 1) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
-                when '{{ test_configuration.anomaly_direction }}' = 'spike' then metric_value
-                else min_metric_value 
-            end as min_value,
-            case
-                when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' = 'drop' then
-                    lagInFrame(metric_value, 1) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
-                when is_anomalous = TRUE and '{{ test_configuration.anomaly_direction }}' != 'drop' then
-                    lagInFrame(max_metric_value, 1) over (partition by full_table_name, column_name, metric_name, dimension, dimension_value, bucket_seasonality order by bucket_end)
-                when '{{ test_configuration.anomaly_direction }}' = 'drop' then metric_value
-                else max_metric_value 
-            end as max_value,
-            bucket_start as start_time,
-            bucket_end as end_time,
-            *
-        from anomaly_scores_with_is_anomalous
-        order by bucket_end, dimension_value
-    )
+      select * from final_results
+      where {{ test_configuration.exclude_final_results }}
+    {%- endset -%}
+    {{- return(anomaly_query) -}}
 {% endmacro %}
 
 {%- macro set_directional_anomaly(anomaly_direction, anomaly_score, sensitivity) -%}

macros/utils/data_types/cast_column.sql

@@ -23,11 +23,9 @@
 
 {%- macro clickhouse__edr_cast_as_timestamp(timestamp_field) -%}
     coalesce(
-        toDateTime64(
-            left(splitByChar('+', toString({{ timestamp_field }}))[1], 23),
-            3
-        ),
-        toDateTime64('1970-01-01 00:00:00', 3)
+        toDateTime(
+            left(splitByChar('+', toString({{ timestamp_field }}))[1], 19)),
+        toDateTime('1970-01-01 00:00:00')
     )
 {%- endmacro -%}
 

macros/utils/data_types/data_type.sql

@@ -38,7 +38,7 @@
 {% endmacro %}
 
 {% macro clickhouse__edr_type_string() %}
-    {% do return("varchar(4096)") %}
+    {% do return("String") %}
 {% endmacro %}
 
 {% macro snowflake__edr_type_string() %}

macros/utils/data_types/data_type_list.sql

@@ -135,7 +135,7 @@
 {% macro clickhouse__data_type_list(data_type) %}
     {% set string_list = ['String', 'FixedString', 'LowCardinality(String)'] | list %}
     {% set numeric_list = ['Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64', 'Decimal', 'Decimal32', 'Decimal64', 'Decimal128'] | list %}
-    {% set timestamp_list = ['DateTime', 'DateTime64', 'Date', 'Date32'] | list %}
+    {% set timestamp_list = ['DateTime', 'Date', 'Date32'] | list %}
     {% set boolean_list = ["UInt8","Bool"] | list %}
 
     {%- if data_type == 'string' %}

macros/utils/data_types/null_as.sql

@@ -1,11 +1,27 @@
 {%- macro null_int() -%}
+    {{ return(adapter.dispatch('null_int', 'elementary')()) }}
+{%- endmacro -%}
+
+{%- macro default__null_int() -%}
     cast(null as {{ elementary.edr_type_int() }})
 {%- endmacro -%}
 
+{%- macro clickhouse__null_int() -%}
+    cast(null as Nullable({{ elementary.edr_type_int() }}))
+{%- endmacro -%}
+
 {%- macro null_timestamp() -%}
+    {{ return(adapter.dispatch('null_timestamp', 'elementary')()) }}
+{%- endmacro -%}
+
+{%- macro default__null_timestamp() -%}
     cast(null as {{ elementary.edr_type_timestamp() }})
 {%- endmacro -%}
 
+{%- macro clickhouse__null_timestamp() -%}
+    cast(null as Nullable({{ elementary.edr_type_timestamp() }}))
+{%- endmacro -%}
+
 {%- macro null_float() -%}
     cast(null as {{ elementary.edr_type_float() }})
 {%- endmacro -%}