only anomaly dim values

Author: NoyaArie

integration_tests/tests/test_dimension_anomalies.py

@@ -87,13 +87,6 @@ def test_anomalous_dimension_anomalies(test_id: str, dbt_project: DbtProject):
         for superhero in ["Superman", "Superman", "Superman", "Spiderman"]
     ]
 
-    test_result = dbt_project.test(test_id, DBT_TEST_NAME, DBT_TEST_ARGS, data=data)
-
-    assert test_result["status"] == "pass"
-
-    anomaly_test_points = get_latest_anomaly_test_points(dbt_project, test_id)
-    assert len(anomaly_test_points) == 0
-
     data += [
         {
             TIMESTAMP_COLUMN: cur_date.strftime(DATE_FORMAT),
@@ -108,23 +101,17 @@ def test_anomalous_dimension_anomalies(test_id: str, dbt_project: DbtProject):
 
     anomaly_test_points = get_latest_anomaly_test_points(dbt_project, test_id)
 
-    # Only test with anomalies are stored in the test points
+    # Only dimension values with anomalies are stored in the test points
     dimension_values = set([x["dimension_value"] for x in anomaly_test_points])
-    assert len(dimension_values) == 2
-    assert "Superman" in dimension_values
-    assert "Spiderman" in dimension_values
 
     superman_anomaly_test_points = [
         x for x in anomaly_test_points if x["dimension_value"] == "Superman"
     ]
-    assert len(superman_anomaly_test_points) == 13
-    assert any(x["is_anomalous"] for x in superman_anomaly_test_points)
 
-    spiderman_anomaly_test_points = [
-        x for x in anomaly_test_points if x["dimension_value"] == "Spiderman"
-    ]
-    assert len(spiderman_anomaly_test_points) == 13
-    assert not any(x["is_anomalous"] for x in spiderman_anomaly_test_points)
+    assert len(dimension_values) == 1
+    assert "Superman" in dimension_values
+    assert len(anomaly_test_points) == len(superman_anomaly_test_points)
+    assert any(x["is_anomalous"] for x in superman_anomaly_test_points)
 
 
 # Anomalies currently not supported on ClickHouse

macros/edr/tests/test_dimension_anomalies.sql

@@ -40,7 +40,7 @@
                                                                                                    detection_period=detection_period,
                                                                                                    training_period=training_period,
                                                                                                    exclude_final_results=exclude_final_results) %}
-        
+
         {%- if not test_configuration %}
             {{ exceptions.raise_compiler_error("Failed to create test configuration dict for test `{}`".format(test_table_name)) }}
         {%- endif %}
@@ -71,7 +71,7 @@
         {{ elementary.test_log('end', full_table_name) }}
 
         {% set flattened_test = elementary.flatten_test(context["model"]) %}
-        {% set anomalous_rows_sql = elementary.get_anomaly_query(flattened_test) %}
+        {% set anomalous_rows_sql = elementary.get_anomaly_query_for_dimension_anomalies(flattened_test) %}
         {% do elementary.store_metrics_table_in_cache() %}
         {% do elementary.store_anomaly_test_results(flattened_test, anomalous_rows_sql) %}
 

macros/edr/tests/test_utils/get_anomaly_query.sql

@@ -1,27 +1,26 @@
 {%- macro get_anomaly_query(flattened_test=none) -%}
-  {%- set anomaly_scores_query -%}
+  {%- set query -%}
     select * from ({{ elementary.get_read_anomaly_scores_query(flattened_test) }}) results
-  {%- endset -%}
-
-  {% set anomalies_count_query -%}
-    select count(*) as anomalies_count
-    from ({{ anomaly_scores_query }})
     where is_anomalous = true
   {%- endset -%}
+  {{- return(query) -}}
+{%- endmacro -%}
 
-  {% set non_anomalies_query -%}
-    {{ anomaly_scores_query }}
-    where is_anomalous = true
+{%- macro get_anomaly_query_for_dimension_anomalies(flattened_test=none) -%}
+  {%- set anomaly_scores_query -%}
+    select distinct dimension_value
+    from ({{ elementary.get_read_anomaly_scores_query(flattened_test) }}) results
+    where dimension is not null
+    and dimension_value is not null
+    and is_anomalous = true
   {%- endset -%}
 
-  {% set anomalies_count_result = elementary.agate_to_dicts(elementary.run_query(anomalies_count_query)) %}
-  {% set anomalies_count = anomalies_count_result[0]['anomalies_count'] %}
+  {% set anomalous_dimension_values_query -%}
+    select * from ({{ elementary.get_read_anomaly_scores_query(flattened_test) }}) results
+    where dimension_value in ({{ anomaly_scores_query }})
+  {%- endset -%}
 
-  {% if anomalies_count > 0 %}
-    {{- return(anomaly_scores_query) -}}
-  {% else %}
-    {{- return(non_anomalies_query) -}}
-  {% endif %}
+  {{- return(anomalous_dimension_values_query) -}}
 {%- endmacro -%}
 
 {% macro get_read_anomaly_scores_query(flattened_test=none) %}