Added time limit to the alerts query

Author: oravi

monitor/data_monitoring.py

@@ -4,6 +4,7 @@
 from config.config import Config
 from utils.log import get_logger
 import json
+from alive_progress import alive_it
 
 logger = get_logger(__name__)
 FILE_DIR = os.path.dirname(__file__)
@@ -52,7 +53,8 @@ def _send_to_slack(self, alerts: [Alert]) -> None:
         slack_webhook = self.config.slack_notification_webhook
         if slack_webhook is not None:
             sent_alerts = []
-            for alert in alerts:
+            alerts_with_progress_bar = alive_it(alerts, title="Sending alerts")
+            for alert in alerts_with_progress_bar:
                 alert.send_to_slack(slack_webhook, self.config.is_slack_workflow)
                 sent_alerts.append(alert.id)
 

monitor/dbt_project/macros/get_alerts_time_limit.sql

@@ -0,0 +1,5 @@
+{% macro get_alerts_time_limit(days_back=31) %}
+    {% set today = dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) %}
+    {% set datetime_limit = dbt_utils.dateadd('day', days_back * -1, today) %}
+    {{ return(elementary.cast_as_timestamp(datetime_limit)) }}
+{% endmacro %}

monitor/dbt_project/macros/get_new_alerts.sql

@@ -1,12 +1,12 @@
 {% macro get_new_alerts() %}
     -- depends_on: {{ ref('alerts') }}
+    {% set current_date = dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) %}
     {% set select_new_alerts_query %}
         SELECT alert_id, detected_at, database_name, schema_name, table_name, column_name, alert_type, sub_type,
                alert_description
         FROM {{ ref('alerts') }}
-        WHERE alert_sent = FALSE
+        WHERE alert_sent = FALSE and detected_at >= {{ get_alerts_time_limit() }}
     {% endset %}
-
     {% set results = run_query(select_new_alerts_query) %}
     {% set new_alerts = [] %}
     {% for result in results %}

monitor/dbt_project/macros/update_sent_alerts.sql

@@ -3,7 +3,8 @@
     {% if alert_ids %}
         {% set update_sent_alerts_query %}
             UPDATE {{ ref('alerts') }} set alert_sent = TRUE
-            WHERE alert_id IN {{ elementary.strings_list_to_tuple(alert_ids) }} and alert_sent = FALSE
+            WHERE alert_id IN {{ elementary.strings_list_to_tuple(alert_ids) }} and alert_sent = FALSE and
+                detected_at >= {{ get_alerts_time_limit() }}
         {% endset %}
         {% set results = dbt_utils.get_query_results_as_dict(update_sent_alerts_query) %}
         {% do elementary.edr_log(results) %}

monitor/dbt_project/packages.yml

@@ -1,5 +1,5 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: [">=0.8.0", "<0.9.0"]
+    version: 0.7.6
   - package: elementary-data/elementary
     version: 0.3.11