add india messages

Author: peterdudfield

terraform/modules/services/airflow/dags/india/elasticbeanstalk.py

@@ -6,7 +6,7 @@
 from airflow.operators.latest_only import LatestOnlyOperator
 from airflow.operators.python import PythonOperator
 from utils.elastic_beanstalk import scale_elastic_beanstalk_instance
-from utils.slack import on_failure_callback
+from utils.slack import slack_message_callback_no_action_required
 
 default_args = {
     "owner": "airflow",
@@ -45,15 +45,15 @@
             python_callable=scale_elastic_beanstalk_instance,
             op_kwargs={"name": name, "number_of_instances": 2, "sleep_seconds": 60 * 5},
             task_concurrency=2,
-            on_failure_callback=on_failure_callback,
+            on_failure_callback=slack_message_callback_no_action_required,
         )
 
         elb_1 = PythonOperator(
             task_id=f"scale_elb_1_{name}",
             python_callable=scale_elastic_beanstalk_instance,
             op_kwargs={"name": name, "number_of_instances": 1},
             task_concurrency=2,
-            on_failure_callback=on_failure_callback,
+            on_failure_callback=slack_message_callback_no_action_required,
         )
 
         latest_only >> elb_2 >> elb_1

terraform/modules/services/airflow/dags/india/forecast-site-dag.py

@@ -2,7 +2,7 @@
 from datetime import datetime, timedelta, timezone
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
-from utils.slack import on_failure_callback
+from utils.slack import slack_message_callback
 
 from airflow.operators.latest_only import LatestOnlyOperator
 
@@ -24,6 +24,17 @@
 
 region = "india"
 
+forecast_ruvnl_error_message = (
+    "❌ The task {{ ti.task_id }} failed. "
+    "This would ideally be fixed before for DA actions at 09.00 IST"
+    "Please see run book for appropriate actions."
+)
+
+forecast_ad_error_message = (
+    "❌ The task {{ ti.task_id }} failed.  "
+    "Please see run book for appropriate actions. "
+)
+
 # hour the forecast can run, not include 7,8,19,20
 hours = "0,1,2,3,4,5,6,9,10,11,12,13,14,15,16,17,18,21,22,23"
 
@@ -51,7 +62,7 @@
                 "assignPublicIp": "ENABLED",
             },
         },
-        on_failure_callback=on_failure_callback,
+        on_failure_callback=slack_message_callback(forecast_ruvnl_error_message),
         task_concurrency=10,
         awslogs_group="/aws/ecs/forecast/forecast",
         awslogs_stream_prefix="streaming/forecast-forecast",
@@ -84,7 +95,7 @@
                 "assignPublicIp": "ENABLED",
             },
         },
-        on_failure_callback=on_failure_callback,
+        on_failure_callback=slack_message_callback(forecast_ad_error_message),
         task_concurrency=10,
         awslogs_group="/aws/ecs/forecast/forecast-ad",
         awslogs_stream_prefix="streaming/forecast-ad-forecast",

terraform/modules/services/airflow/dags/india/nwp-dag.py

@@ -4,7 +4,7 @@
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
 
 from airflow.operators.latest_only import LatestOnlyOperator
-from utils.slack import on_failure_callback
+from utils.slack import slack_message_callback
 from utils.s3 import determine_latest_zarr
 
 default_args = {
@@ -25,6 +25,29 @@
 
 region = "india"
 
+nwp_metoffice_error_message = (
+    "❌ The task {{ ti.task_id }} failed."
+    "The forecast will continue running until it runs out of data."
+    "which doesnt need Metoffice data. "
+    "Metoffice status link is <https://datahub.metoffice.gov.uk/support/service-status|here> "
+    "Please see run book for appropriate actions. "
+)
+
+nwp_ecmwf_error_message = (
+    "❌ The task {{ ti.task_id }} failed."
+    "The forecast will continue running until it runs out of data. "
+    "ECMWF status link is <https://www.nco.ncep.noaa.gov/pmb/nwprod/prodstat/|here> "
+    "Please see run book for appropriate actions. "
+)
+
+nwp_gfs_error_message = (
+    "❌ The task {{ ti.task_id }} failed."
+    "The forecast will continue running until it runs out of data. "
+    "ECMWF status link is <https://status.ecmwf.int/|here> "
+    "Please see run book for appropriate actions. "
+)
+
+
 with DAG(
     f"{region}-nwp-consumer",
     schedule_interval="0 * * * *",
@@ -50,6 +73,7 @@
             },
         },
         task_concurrency=10,
+        on_failure_callback=slack_message_callback(nwp_ecmwf_error_message),
         awslogs_group="/aws/ecs/consumer/nwp-consumer-ecmwf-india",
         awslogs_stream_prefix="streaming/nwp-consumer-ecmwf-india-consumer",
         awslogs_region="ap-south-1",
@@ -69,6 +93,7 @@
             },
         },
         task_concurrency=10,
+        on_failure_callback=slack_message_callback(nwp_gfs_error_message),
         awslogs_group="/aws/ecs/consumer/nwp-consumer-gfs-india",
         awslogs_stream_prefix="streaming/nwp-consumer-gfs-india-consumer",
         awslogs_region="ap-south-1",
@@ -88,6 +113,7 @@
             },
         },
         task_concurrency=10,
+        on_failure_callback=slack_message_callback(nwp_metoffice_error_message),
         awslogs_group="/aws/ecs/consumer/nwp-consumer-metoffice-india",
         awslogs_stream_prefix="streaming/nwp-consumer-metoffice-india-consumer",
         awslogs_region="ap-south-1",

terraform/modules/services/airflow/dags/india/runvnl-data-dag.py

@@ -2,7 +2,7 @@
 from datetime import datetime, timedelta, timezone
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
-from utils.slack import on_failure_callback
+from utils.slack import slack_message_callback_no_action_required
 
 from airflow.operators.latest_only import LatestOnlyOperator
 
@@ -48,7 +48,7 @@
                 "assignPublicIp": "ENABLED",
             },
         },
-        on_failure_callback=on_failure_callback,
+        on_failure_callback=slack_message_callback_no_action_required,
         task_concurrency=10,
         awslogs_group="/aws/ecs/consumer/runvl-consumer",
         awslogs_stream_prefix="streaming/runvl-consumer-consumer",

terraform/modules/services/airflow/dags/india/satellite-dag.py

@@ -4,7 +4,7 @@
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
 
 from airflow.operators.latest_only import LatestOnlyOperator
-from utils.slack import on_failure_callback
+from utils.slack import slack_message_callback
 
 default_args = {
     "owner": "airflow",
@@ -24,6 +24,13 @@
 
 # Tasks can still be defined in terraform, or defined here
 
+satellite_error_message = (
+    "❌ The task {{ ti.task_id }} failed."
+    "EUMETSAT status links are <https://uns.eumetsat.int/uns/|here> "
+    "and <https://masif.eumetsat.int/ossi/webpages/level3.html?ossi_level3_filename=seviri_rss_hr.html&ossi_level2_filename=seviri_rss.html|here>. "
+    "Please see run book for appropriate actions. "
+)
+
 region = "india"
 
 with DAG(
@@ -52,7 +59,7 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback,
+        on_failure_callback=slack_message_callback(satellite_error_message),
         awslogs_group="/aws/ecs/consumer/sat-consumer",
         awslogs_stream_prefix="streaming/sat-consumer-consumer",
         awslogs_region="ap-south-1",

terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py

@@ -33,13 +33,13 @@
 
 forecast_pvnet_da_error_message = (
     "❌ The task {{ ti.task_id }} failed. "
-    "This would ideally before for DA actions at 09.00"
+    "This would ideally be fixed before for DA actions at 09.00"
     "Please see run book for appropriate actions."
 )
 
 forecast_ecmwf_error_message = (
     "❌ The task {{ ti.task_id }} failed. This is only run after the main PVnet has failed. "
-    "We have about 6 hours before this is needed. "
+    "We have about 6 hours before the blend services need this. "
     "Please see run book for appropriate actions. "
 )
 

terraform/modules/services/airflow/dags/utils/slack.py

@@ -13,11 +13,21 @@
         )
     ]
 
+slack_message_callback_no_action_required = [
+        send_slack_notification(
+            text="⚠️ The task {{ ti.task_id }} failed,"
+    " but its ok. No out of hours support is required.",
+            channel=f"tech-ops-airflow-{env}",
+            username="Airflow",
+        )
+    ]
+
 def slack_message_callback(message):
     return [
         send_slack_notification(
             text=message,
             channel=f"tech-ops-airflow-{env}",
             username="Airflow",
         )
-    ]
+    ]
+