Skip to content

Commit d0bc714

Browse files
NoyaArieNoyaArie
authored
Ele 4942 dimension anomalies visualization (#1992)
* no table view for dimension test * lint * update dbt package version
1 parent 792cd1b commit d0bc714

File tree

3 files changed

+10
-92
lines changed

3 files changed

+10
-92
lines changed

‎elementary/monitor/dbt_project/macros/get_test_results.sql‎

Lines changed: 4 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
{%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%}
66
{% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %}
7-
87
{% set select_test_results %}
98
with test_results as (
109
{{ elementary_cli.current_tests_run_results_query(days_back=days_back) }}
@@ -19,7 +18,7 @@
1918
from test_results
2019
)
2120

22-
select
21+
select
2322
test_results.id,
2423
test_results.invocation_id,
2524
test_results.test_execution_id,
@@ -70,7 +69,7 @@
7069
{% endset %}
7170

7271
{% set valid_ids_query %}
73-
select distinct id
72+
select distinct id
7473
from {{ ordered_test_results_relation }}
7574
where invocations_rank_index = 1
7675
{% endset %}
@@ -99,54 +98,12 @@
9998

10099
{%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%}
101100
{% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %}
102-
{# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #}
103-
{# Currently we show only the anomalous for all of the dimensions. #}
104-
{% if test.test_sub_type == 'dimension' or test_params.dimensions %}
105-
{% if test.test_sub_type == 'dimension' %}
106-
{% set metric_name = 'row_count' %}
107-
{% elif test_params.dimensions %}
108-
{% set metric_name = test.test_sub_type %}
109-
{% endif %}
110-
{% set anomalous_rows = [] %}
111-
{% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %}
112-
{% for row in test_rows_sample %}
113-
{% set anomalous_row = {
114-
'anomalous_value_timestamp': row['end_time'],
115-
'anomalous_value_' ~ metric_name: row['value'],
116-
'anomalous_value_average_' ~ metric_name: row['average'] | round(1)
117-
} %}
118-
{% set dimensions = row['dimension'].split('; ') %}
119-
{% set diemsions_values = row['dimension_value'].split('; ') %}
120-
{% for index in range(dimensions | length) %}
121-
{% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %}
122-
{% endfor %}
123-
{% if loop.last %}
124-
{# Adding dimensions to the headers #}
125-
{% for index in range(dimensions | length) %}
126-
{% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %}
127-
{% endfor %}
128-
{% endif %}
129-
{% if row['is_anomalous'] %}
130-
{% do anomalous_rows.append(anomalous_row) %}
131-
{% endif %}
132-
{% endfor %}
133-
{# Adding the rest of the static headers (metrics headers) #}
134-
{% do headers.extend([
135-
{'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'},
136-
{'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'}
137-
]) %}
138-
{% set test_rows_sample = {
139-
'headers': headers,
140-
'test_rows_sample': anomalous_rows
141-
} %}
142-
{% endif %}
143101
{%- endif -%}
144102
{% endif %}
145103
{# Adding sample data to test results #}
146104
{% do test.update({"sample_data": test_rows_sample}) %}
147105
{% do test_results.append(test) %}
148106
{%- endfor -%}
149-
150107
{% do return(test_results) %}
151108
{%- endmacro -%}
152109

@@ -242,7 +199,7 @@
242199
FROM {{ ref('elementary', 'elementary_test_results') }} etr
243200
JOIN {{ ref('elementary', 'dbt_tests') }} dt ON etr.test_unique_id = dt.unique_id
244201
LEFT JOIN (
245-
SELECT
202+
SELECT
246203
min(detected_at) AS first_time_occurred,
247204
test_unique_id
248205
FROM {{ ref('elementary', 'elementary_test_results') }}
@@ -272,7 +229,7 @@
272229
{% endset %}
273230

274231
{% set valid_ids_query %}
275-
select distinct id
232+
select distinct id
276233
from {{ ordered_test_results_relation }}
277234
where invocations_rank_index = 1
278235
{% endset %}
@@ -301,47 +258,6 @@
301258

302259
{%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%}
303260
{% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %}
304-
{# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #}
305-
{# Currently we show only the anomalous for all of the dimensions. #}
306-
{% if test.test_sub_type == 'dimension' or test_params.dimensions %}
307-
{% if test.test_sub_type == 'dimension' %}
308-
{% set metric_name = 'row_count' %}
309-
{% elif test_params.dimensions %}
310-
{% set metric_name = test.test_sub_type %}
311-
{% endif %}
312-
{% set anomalous_rows = [] %}
313-
{% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %}
314-
{% for row in test_rows_sample %}
315-
{% set anomalous_row = {
316-
'anomalous_value_timestamp': row['end_time'],
317-
'anomalous_value_' ~ metric_name: row['value'],
318-
'anomalous_value_average_' ~ metric_name: row['average'] | round(1)
319-
} %}
320-
{% set dimensions = row['dimension'].split('; ') %}
321-
{% set diemsions_values = row['dimension_value'].split('; ') %}
322-
{% for index in range(dimensions | length) %}
323-
{% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %}
324-
{% endfor %}
325-
{% if loop.last %}
326-
{# Adding dimensions to the headers #}
327-
{% for index in range(dimensions | length) %}
328-
{% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %}
329-
{% endfor %}
330-
{% endif %}
331-
{% if row['is_anomalous'] %}
332-
{% do anomalous_rows.append(anomalous_row) %}
333-
{% endif %}
334-
{% endfor %}
335-
{# Adding the rest of the static headers (metrics headers) #}
336-
{% do headers.extend([
337-
{'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'},
338-
{'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'}
339-
]) %}
340-
{% set test_rows_sample = {
341-
'headers': headers,
342-
'test_rows_sample': anomalous_rows
343-
} %}
344-
{% endif %}
345261
{%- endif -%}
346262
{% endif %}
347263
{# Adding sample data to test results #}

‎elementary/monitor/dbt_project/package-lock.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
packages:
2-
- package: dbt-labs/dbt_utils
2+
- name: dbt_utils
3+
package: dbt-labs/dbt_utils
34
version: 0.8.6
45
- git: https://github.com/elementary-data/dbt-data-reliability.git
5-
revision: 257974756e195140204ac4a76278a26acf78a27f
6-
sha1_hash: 0bac331a6cf2d739e67a64cf27532d9502175e37
6+
name: elementary
7+
revision: b46d220a21487a926f1cb50f5dc4c77e321f74c2
8+
sha1_hash: 01feb82f69ff6c22843c3ccc6dce57c355417a9e

‎elementary/monitor/dbt_project/packages.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ packages:
22
- package: dbt-labs/dbt_utils
33
version: [">=0.8.0", "<0.9.0"]
44
- git: https://github.com/elementary-data/dbt-data-reliability.git
5-
revision: 257974756e195140204ac4a76278a26acf78a27f
5+
revision: b46d220a21487a926f1cb50f5dc4c77e321f74c2
66

77
# NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this
88
# commented, so it will be easy to access)

0 commit comments

Comments
 (0)