|
4 | 4 |
|
5 | 5 | {%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} |
6 | 6 | {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %} |
7 | | - |
8 | 7 | {% set select_test_results %} |
9 | 8 | with test_results as ( |
10 | 9 | {{ elementary_cli.current_tests_run_results_query(days_back=days_back) }} |
|
19 | 18 | from test_results |
20 | 19 | ) |
21 | 20 |
|
22 | | - select |
| 21 | + select |
23 | 22 | test_results.id, |
24 | 23 | test_results.invocation_id, |
25 | 24 | test_results.test_execution_id, |
|
70 | 69 | {% endset %} |
71 | 70 |
|
72 | 71 | {% set valid_ids_query %} |
73 | | - select distinct id |
| 72 | + select distinct id |
74 | 73 | from {{ ordered_test_results_relation }} |
75 | 74 | where invocations_rank_index = 1 |
76 | 75 | {% endset %} |
|
99 | 98 |
|
100 | 99 | {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} |
101 | 100 | {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} |
102 | | - {# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #} |
103 | | - {# Currently we show only the anomalous for all of the dimensions. #} |
104 | | - {% if test.test_sub_type == 'dimension' or test_params.dimensions %} |
105 | | - {% if test.test_sub_type == 'dimension' %} |
106 | | - {% set metric_name = 'row_count' %} |
107 | | - {% elif test_params.dimensions %} |
108 | | - {% set metric_name = test.test_sub_type %} |
109 | | - {% endif %} |
110 | | - {% set anomalous_rows = [] %} |
111 | | - {% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %} |
112 | | - {% for row in test_rows_sample %} |
113 | | - {% set anomalous_row = { |
114 | | - 'anomalous_value_timestamp': row['end_time'], |
115 | | - 'anomalous_value_' ~ metric_name: row['value'], |
116 | | - 'anomalous_value_average_' ~ metric_name: row['average'] | round(1) |
117 | | - } %} |
118 | | - {% set dimensions = row['dimension'].split('; ') %} |
119 | | - {% set diemsions_values = row['dimension_value'].split('; ') %} |
120 | | - {% for index in range(dimensions | length) %} |
121 | | - {% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %} |
122 | | - {% endfor %} |
123 | | - {% if loop.last %} |
124 | | - {# Adding dimensions to the headers #} |
125 | | - {% for index in range(dimensions | length) %} |
126 | | - {% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %} |
127 | | - {% endfor %} |
128 | | - {% endif %} |
129 | | - {% if row['is_anomalous'] %} |
130 | | - {% do anomalous_rows.append(anomalous_row) %} |
131 | | - {% endif %} |
132 | | - {% endfor %} |
133 | | - {# Adding the rest of the static headers (metrics headers) #} |
134 | | - {% do headers.extend([ |
135 | | - {'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'}, |
136 | | - {'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'} |
137 | | - ]) %} |
138 | | - {% set test_rows_sample = { |
139 | | - 'headers': headers, |
140 | | - 'test_rows_sample': anomalous_rows |
141 | | - } %} |
142 | | - {% endif %} |
143 | 101 | {%- endif -%} |
144 | 102 | {% endif %} |
145 | 103 | {# Adding sample data to test results #} |
146 | 104 | {% do test.update({"sample_data": test_rows_sample}) %} |
147 | 105 | {% do test_results.append(test) %} |
148 | 106 | {%- endfor -%} |
149 | | - |
150 | 107 | {% do return(test_results) %} |
151 | 108 | {%- endmacro -%} |
152 | 109 |
|
|
242 | 199 | FROM {{ ref('elementary', 'elementary_test_results') }} etr |
243 | 200 | JOIN {{ ref('elementary', 'dbt_tests') }} dt ON etr.test_unique_id = dt.unique_id |
244 | 201 | LEFT JOIN ( |
245 | | - SELECT |
| 202 | + SELECT |
246 | 203 | min(detected_at) AS first_time_occurred, |
247 | 204 | test_unique_id |
248 | 205 | FROM {{ ref('elementary', 'elementary_test_results') }} |
|
272 | 229 | {% endset %} |
273 | 230 |
|
274 | 231 | {% set valid_ids_query %} |
275 | | - select distinct id |
| 232 | + select distinct id |
276 | 233 | from {{ ordered_test_results_relation }} |
277 | 234 | where invocations_rank_index = 1 |
278 | 235 | {% endset %} |
|
301 | 258 |
|
302 | 259 | {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} |
303 | 260 | {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} |
304 | | - {# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #} |
305 | | - {# Currently we show only the anomalous for all of the dimensions. #} |
306 | | - {% if test.test_sub_type == 'dimension' or test_params.dimensions %} |
307 | | - {% if test.test_sub_type == 'dimension' %} |
308 | | - {% set metric_name = 'row_count' %} |
309 | | - {% elif test_params.dimensions %} |
310 | | - {% set metric_name = test.test_sub_type %} |
311 | | - {% endif %} |
312 | | - {% set anomalous_rows = [] %} |
313 | | - {% set headers = [{'id': 'anomalous_value_timestamp', 'display_name': 'timestamp', 'type': 'date'}] %} |
314 | | - {% for row in test_rows_sample %} |
315 | | - {% set anomalous_row = { |
316 | | - 'anomalous_value_timestamp': row['end_time'], |
317 | | - 'anomalous_value_' ~ metric_name: row['value'], |
318 | | - 'anomalous_value_average_' ~ metric_name: row['average'] | round(1) |
319 | | - } %} |
320 | | - {% set dimensions = row['dimension'].split('; ') %} |
321 | | - {% set diemsions_values = row['dimension_value'].split('; ') %} |
322 | | - {% for index in range(dimensions | length) %} |
323 | | - {% do anomalous_row.update({dimensions[index]: diemsions_values[index]}) %} |
324 | | - {% endfor %} |
325 | | - {% if loop.last %} |
326 | | - {# Adding dimensions to the headers #} |
327 | | - {% for index in range(dimensions | length) %} |
328 | | - {% do headers.append({'id': dimensions[index], 'display_name': dimensions[index], 'type': 'str'},) %} |
329 | | - {% endfor %} |
330 | | - {% endif %} |
331 | | - {% if row['is_anomalous'] %} |
332 | | - {% do anomalous_rows.append(anomalous_row) %} |
333 | | - {% endif %} |
334 | | - {% endfor %} |
335 | | - {# Adding the rest of the static headers (metrics headers) #} |
336 | | - {% do headers.extend([ |
337 | | - {'id': 'anomalous_value_' ~ metric_name, 'display_name': ' '.join(metric_name.split('_')), 'type': 'int'}, |
338 | | - {'id': 'anomalous_value_average_' ~ metric_name, 'display_name': 'average ' ~ ' '.join(metric_name.split('_')), 'type': 'int'} |
339 | | - ]) %} |
340 | | - {% set test_rows_sample = { |
341 | | - 'headers': headers, |
342 | | - 'test_rows_sample': anomalous_rows |
343 | | - } %} |
344 | | - {% endif %} |
345 | 261 | {%- endif -%} |
346 | 262 | {% endif %} |
347 | 263 | {# Adding sample data to test results #} |
|
0 commit comments