Skip to content

Ele 4942 dimension anomalies visualization #1992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NoyaArie merged 4 commits into from
Aug 27, 2025
Merged

Ele 4942 dimension anomalies visualization #1992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0a93016
no table view for dimension test
NoyaArie Aug 21, 2025
0555e31
lint
NoyaArie Aug 21, 2025
7e3d23e
Merge branch 'master' into ele-4942-dimension-anomalies-visualization
NoyaArie Aug 27, 2025
29daa5c
update dbt package version
NoyaArie Aug 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 4 additions & 88 deletions elementary/monitor/dbt_project/macros/get_test_results.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

{%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%}
{% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %}

{% set select_test_results %}
with test_results as (
{{ elementary_cli.current_tests_run_results_query(days_back=days_back) }}
Expand All @@ -19,7 +18,7 @@
from test_results
)

select
select
test_results.id,
test_results.invocation_id,
test_results.test_execution_id,
Expand Down Expand Up @@ -70,7 +69,7 @@
{% endset %}

{% set valid_ids_query %}
select distinct id
select distinct id
from {{ ordered_test_results_relation }}
where invocations_rank_index = 1
{% endset %}
Expand Down Expand Up @@ -99,54 +98,12 @@

{%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%}
{% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %}
{# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #}
{# Currently we show only the anomalous for all of the dimensions. #}
{% if test.test_sub_type == 'dimension' or test_params.dimensions %}
{% if test.test_sub_type == 'dimension' %}
{% set metric_name = 'row_count' %}
{% elif test_params.dimensions %}
{% set metric_name = test.test_sub_type %}
{% endif %}
{% set anomalous_rows = [] %}
{% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %}
{% for row in test_rows_sample %}
{% set anomalous_row = {
'anomalous_value_timestamp': row['end_time'],
'anomalous_value_' ~ metric_name: row['value'],
'anomalous_value_average_' ~ metric_name: row['average'] | round(1)
} %}
{% set dimensions = row['dimension'].split('; ') %}
{% set diemsions_values = row['dimension_value'].split('; ') %}
{% for index in range(dimensions | length) %}
{% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %}
{% endfor %}
{% if loop.last %}
{# Adding dimensions to the headers #}
{% for index in range(dimensions | length) %}
{% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %}
{% endfor %}
{% endif %}
{% if row['is_anomalous'] %}
{% do anomalous_rows.append(anomalous_row) %}
{% endif %}
{% endfor %}
{# Adding the rest of the static headers (metrics headers) #}
{% do headers.extend([
{'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'},
{'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'}
]) %}
{% set test_rows_sample = {
'headers': headers,
'test_rows_sample': anomalous_rows
} %}
{% endif %}
{%- endif -%}
{% endif %}
{# Adding sample data to test results #}
{% do test.update({"sample_data": test_rows_sample}) %}
{% do test_results.append(test) %}
{%- endfor -%}

{% do return(test_results) %}
{%- endmacro -%}

Expand Down Expand Up @@ -242,7 +199,7 @@
FROM {{ ref('elementary', 'elementary_test_results') }} etr
JOIN {{ ref('elementary', 'dbt_tests') }} dt ON etr.test_unique_id = dt.unique_id
LEFT JOIN (
SELECT
SELECT
min(detected_at) AS first_time_occurred,
test_unique_id
FROM {{ ref('elementary', 'elementary_test_results') }}
Expand Down Expand Up @@ -272,7 +229,7 @@
{% endset %}

{% set valid_ids_query %}
select distinct id
select distinct id
from {{ ordered_test_results_relation }}
where invocations_rank_index = 1
{% endset %}
Expand Down Expand Up @@ -301,47 +258,6 @@

{%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%}
{% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %}
{# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #}
{# Currently we show only the anomalous for all of the dimensions. #}
{% if test.test_sub_type == 'dimension' or test_params.dimensions %}
{% if test.test_sub_type == 'dimension' %}
{% set metric_name = 'row_count' %}
{% elif test_params.dimensions %}
{% set metric_name = test.test_sub_type %}
{% endif %}
{% set anomalous_rows = [] %}
{% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %}
{% for row in test_rows_sample %}
{% set anomalous_row = {
'anomalous_value_timestamp': row['end_time'],
'anomalous_value_' ~ metric_name: row['value'],
'anomalous_value_average_' ~ metric_name: row['average'] | round(1)
} %}
{% set dimensions = row['dimension'].split('; ') %}
{% set diemsions_values = row['dimension_value'].split('; ') %}
{% for index in range(dimensions | length) %}
{% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %}
{% endfor %}
{% if loop.last %}
{# Adding dimensions to the headers #}
{% for index in range(dimensions | length) %}
{% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %}
{% endfor %}
{% endif %}
{% if row['is_anomalous'] %}
{% do anomalous_rows.append(anomalous_row) %}
{% endif %}
{% endfor %}
{# Adding the rest of the static headers (metrics headers) #}
{% do headers.extend([
{'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'},
{'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'}
]) %}
{% set test_rows_sample = {
'headers': headers,
'test_rows_sample': anomalous_rows
} %}
{% endif %}
{%- endif -%}
{% endif %}
{# Adding sample data to test results #}
Expand Down
8 changes: 5 additions & 3 deletions elementary/monitor/dbt_project/package-lock.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
packages:
- package: dbt-labs/dbt_utils
- name: dbt_utils
package: dbt-labs/dbt_utils
version: 0.8.6
- git: https://github.com/elementary-data/dbt-data-reliability.git
revision: 257974756e195140204ac4a76278a26acf78a27f
sha1_hash: 0bac331a6cf2d739e67a64cf27532d9502175e37
name: elementary
revision: b46d220a21487a926f1cb50f5dc4c77e321f74c2
sha1_hash: 01feb82f69ff6c22843c3ccc6dce57c355417a9e
2 changes: 1 addition & 1 deletion elementary/monitor/dbt_project/packages.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ packages:
- package: dbt-labs/dbt_utils
version: [">=0.8.0", "<0.9.0"]
- git: https://github.com/elementary-data/dbt-data-reliability.git
revision: 257974756e195140204ac4a76278a26acf78a27f
revision: b46d220a21487a926f1cb50f5dc4c77e321f74c2

# NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this
# commented, so it will be easy to access)
Expand Down