Add z-score fallback metric tracking for anomaly detection tests

- Add is_zscore_fallback field to anomaly scores query (when training_stddev = 0)
- Propagate is_zscore_fallback through get_read_anomaly_scores_query
- Track zscore_fallback_passed_count and zscore_fallback_failed_count in test results
- Add new fields to elementary_test_results schema

This enables tracking the number of tests that use z-score fallback
(when all training values are identical), with distinction between
passed and failed tests.

Co-Authored-By: Yosef Arbiv <[email protected]>

Author: devin-ai-integration[bot]

macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql

@@ -231,6 +231,10 @@
                     when training_stddev = 0 then 0  -- Stationary data case - valid, all values are identical
                     else (metric_value - training_avg) / (training_stddev)
                 end as anomaly_score,
+                case
+                    when training_stddev is not null and training_set_size > 1 and training_stddev = 0 then TRUE
+                    else FALSE
+                end as is_zscore_fallback,
                 {{ test_configuration.anomaly_sensitivity }} as anomaly_score_threshold,
                 source_value as anomalous_value,
                 {{ elementary.edr_cast_as_timestamp('bucket_start') }} as bucket_start,

macros/edr/data_monitoring/anomaly_detection/store_anomaly_test_results.sql

@@ -48,12 +48,21 @@
       {% endif %}
   {% endset %}
   {% set failures = namespace(data=0) %}
+  {% set zscore_fallback_passed = namespace(data=0) %}
+  {% set zscore_fallback_failed = namespace(data=0) %}
   {% set filtered_anomaly_scores_rows = [] %}
   {% for row in anomaly_scores_rows %}
     {% if row.anomaly_score is not none %}
       {% do filtered_anomaly_scores_rows.append(row) %}
       {% if row.is_anomalous %}
         {% set failures.data = failures.data + 1 %}
+        {% if elementary.insensitive_get_dict_value(row, 'is_zscore_fallback') %}
+          {% set zscore_fallback_failed.data = zscore_fallback_failed.data + 1 %}
+        {% endif %}
+      {% else %}
+        {% if elementary.insensitive_get_dict_value(row, 'is_zscore_fallback') %}
+          {% set zscore_fallback_passed.data = zscore_fallback_passed.data + 1 %}
+        {% endif %}
       {% endif %}
     {% endif %}
   {% endfor %}
@@ -69,7 +78,9 @@
       'test_results_query': test_results_query,
       'test_params': test_params,
       'result_rows': filtered_anomaly_scores_rows,
-      'failures': failures.data
+      'failures': failures.data,
+      'zscore_fallback_passed_count': zscore_fallback_passed.data,
+      'zscore_fallback_failed_count': zscore_fallback_failed.data
   } %}
   {% set elementary_test_row = elementary.get_dbt_test_result_row(flattened_test) %}
   {% do elementary_test_row.update(test_result_dict) %}

macros/edr/system/system_utils/empty_table.sql

@@ -27,7 +27,9 @@
     ('test_short_name', 'string'),
     ('test_alias', 'string'),
     ('result_rows', 'long_string'),
-    ('failed_row_count', 'bigint')
+    ('failed_row_count', 'bigint'),
+    ('zscore_fallback_passed_count', 'bigint'),
+    ('zscore_fallback_failed_count', 'bigint')
     ]) }}
 {% endmacro %}
 

macros/edr/tests/test_utils/get_anomaly_query.sql

@@ -43,6 +43,7 @@
             column_name,
             metric_name,
             anomaly_score,
+            is_zscore_fallback,
             anomaly_score_threshold,
             anomalous_value,
             bucket_start,