diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index a3a18749a..6f59e3b02 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -4,7 +4,6 @@ {%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %} - {% set select_test_results %} with test_results as ( {{ elementary_cli.current_tests_run_results_query(days_back=days_back) }} @@ -19,7 +18,7 @@ from test_results ) - select + select test_results.id, test_results.invocation_id, test_results.test_execution_id, @@ -70,7 +69,7 @@ {% endset %} {% set valid_ids_query %} - select distinct id + select distinct id from {{ ordered_test_results_relation }} where invocations_rank_index = 1 {% endset %} @@ -99,54 +98,12 @@ {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #} - {# Currently we show only the anomalous for all of the dimensions. #} - {% if test.test_sub_type == 'dimension' or test_params.dimensions %} - {% if test.test_sub_type == 'dimension' %} - {% set metric_name = 'row_count' %} - {% elif test_params.dimensions %} - {% set metric_name = test.test_sub_type %} - {% endif %} - {% set anomalous_rows = [] %} - {% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %} - {% for row in test_rows_sample %} - {% set anomalous_row = { - 'anomalous_value_timestamp': row['end_time'], - 'anomalous_value_' ~ metric_name: row['value'], - 'anomalous_value_average_' ~ metric_name: row['average'] | round(1) - } %} - {% set dimensions = row['dimension'].split('; ') %} - {% set diemsions_values = row['dimension_value'].split('; ') %} - {% for index in range(dimensions | length) %} - {% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %} - {% endfor %} - {% if loop.last %} - {# Adding dimensions to the headers #} - {% for index in range(dimensions | length) %} - {% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %} - {% endfor %} - {% endif %} - {% if row['is_anomalous'] %} - {% do anomalous_rows.append(anomalous_row) %} - {% endif %} - {% endfor %} - {# Adding the rest of the static headers (metrics headers) #} - {% do headers.extend([ - {'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'}, - {'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'} - ]) %} - {% set test_rows_sample = { - 'headers': headers, - 'test_rows_sample': anomalous_rows - } %} - {% endif %} {%- endif -%} {% endif %} {# Adding sample data to test results #} {% do test.update({"sample_data": test_rows_sample}) %} {% do test_results.append(test) %} {%- endfor -%} - {% do return(test_results) %} {%- endmacro -%} @@ -242,7 +199,7 @@ FROM {{ ref('elementary', 'elementary_test_results') }} etr JOIN {{ ref('elementary', 'dbt_tests') }} dt ON etr.test_unique_id = dt.unique_id LEFT JOIN ( - SELECT + SELECT min(detected_at) AS first_time_occurred, test_unique_id FROM {{ ref('elementary', 'elementary_test_results') }} @@ -272,7 +229,7 @@ {% endset %} {% set valid_ids_query %} - select distinct id + select distinct id from {{ ordered_test_results_relation }} where invocations_rank_index = 1 {% endset %} @@ -301,47 +258,6 @@ {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #} - {# Currently we show only the anomalous for all of the dimensions. #} - {% if test.test_sub_type == 'dimension' or test_params.dimensions %} - {% if test.test_sub_type == 'dimension' %} - {% set metric_name = 'row_count' %} - {% elif test_params.dimensions %} - {% set metric_name = test.test_sub_type %} - {% endif %} - {% set anomalous_rows = [] %} - {% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %} - {% for row in test_rows_sample %} - {% set anomalous_row = { - 'anomalous_value_timestamp': row['end_time'], - 'anomalous_value_' ~ metric_name: row['value'], - 'anomalous_value_average_' ~ metric_name: row['average'] | round(1) - } %} - {% set dimensions = row['dimension'].split('; ') %} - {% set diemsions_values = row['dimension_value'].split('; ') %} - {% for index in range(dimensions | length) %} - {% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %} - {% endfor %} - {% if loop.last %} - {# Adding dimensions to the headers #} - {% for index in range(dimensions | length) %} - {% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %} - {% endfor %} - {% endif %} - {% if row['is_anomalous'] %} - {% do anomalous_rows.append(anomalous_row) %} - {% endif %} - {% endfor %} - {# Adding the rest of the static headers (metrics headers) #} - {% do headers.extend([ - {'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'}, - {'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'} - ]) %} - {% set test_rows_sample = { - 'headers': headers, - 'test_rows_sample': anomalous_rows - } %} - {% endif %} {%- endif -%} {% endif %} {# Adding sample data to test results #} diff --git a/elementary/monitor/dbt_project/package-lock.yml b/elementary/monitor/dbt_project/package-lock.yml index e5ccce93e..0c9a71c65 100644 --- a/elementary/monitor/dbt_project/package-lock.yml +++ b/elementary/monitor/dbt_project/package-lock.yml @@ -1,6 +1,8 @@ packages: - - package: dbt-labs/dbt_utils + - name: dbt_utils + package: dbt-labs/dbt_utils version: 0.8.6 - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: 257974756e195140204ac4a76278a26acf78a27f -sha1_hash: 0bac331a6cf2d739e67a64cf27532d9502175e37 + name: elementary + revision: b46d220a21487a926f1cb50f5dc4c77e321f74c2 +sha1_hash: 01feb82f69ff6c22843c3ccc6dce57c355417a9e diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml index 6bdd98671..bfec975a4 100644 --- a/elementary/monitor/dbt_project/packages.yml +++ b/elementary/monitor/dbt_project/packages.yml @@ -2,7 +2,7 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.8.0", "<0.9.0"] - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: 257974756e195140204ac4a76278a26acf78a27f + revision: b46d220a21487a926f1cb50f5dc4c77e321f74c2 # NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this # commented, so it will be easy to access)