add aws cloudwatch logs to airflow

peterdudfield · peterdudfield · commit 118d77ac9cc5 · 2025-01-27T14:14:36.000Z
diff --git a/terraform/modules/services/airflow/dags/india/forecast-site-dag.py b/terraform/modules/services/airflow/dags/india/forecast-site-dag.py
@@ -47,6 +47,9 @@
         },
         on_failure_callback=on_failure_callback,
      task_concurrency = 10,
+        awslogs_group='/aws/ecs/forecast/forecast',
+        awslogs_stream_prefix='streaming/forecast-forecast',
+        awslogs_region='ap-south-1',
     )
 
     latest_only >> [forecast]
@@ -71,6 +74,9 @@
         },
         on_failure_callback=on_failure_callback,
      task_concurrency = 10,
+        awslogs_group='/aws/ecs/forecast/forecast0ad',
+        awslogs_stream_prefix='streaming/forecast-ad-forecast',
+        awslogs_region='ap-south-1',
     )
 
     latest_only >> [forecast]
diff --git a/terraform/modules/services/airflow/dags/india/nwp-dag.py b/terraform/modules/services/airflow/dags/india/nwp-dag.py
@@ -50,6 +50,9 @@
              },
          },
          task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/nwp-consumer-ecmwf-india',
+        awslogs_stream_prefix='streaming/nwp-consumer-ecmwf-india-consumer',
+        awslogs_region='ap-south-1'
     )
 
     nwp_consumer_gfs = EcsRunTaskOperator(
@@ -66,6 +69,9 @@
              },
          },
          task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/nwp-consumer-gfs-india',
+        awslogs_stream_prefix='streaming/nwp-consumer-gfs-india-consumer',
+        awslogs_region='ap-south-1'
     )
 
     nwp_consumer_metoffice = EcsRunTaskOperator(
@@ -82,6 +88,9 @@
             },
         },
         task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/nwp-consumer-metoffice-india',
+        awslogs_stream_prefix='streaming/nwp-consumer-metoffice-india-consumer',
+        awslogs_region='ap-south-1'
     )
     rename_zarr_metoffice = determine_latest_zarr.override(
             task_id="determine_latest_zarr_metoffice",
diff --git a/terraform/modules/services/airflow/dags/india/runvnl-data-dag.py b/terraform/modules/services/airflow/dags/india/runvnl-data-dag.py
@@ -50,6 +50,9 @@
         },
         on_failure_callback=on_failure_callback,
         task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/runvl-consumer',
+        awslogs_stream_prefix='streaming/runvl-consumer-consumer',
+        awslogs_region='ap-south-1'
     )
 
 
diff --git a/terraform/modules/services/airflow/dags/india/satellite-dag.py b/terraform/modules/services/airflow/dags/india/satellite-dag.py
@@ -52,7 +52,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/consumer/sat-consumer',
+        awslogs_stream_prefix='streaming/sat-consumer-consumer',
+        awslogs_region='ap-south-1'
     )
 
     latest_only >> sat_consumer
diff --git a/terraform/modules/services/airflow/dags/uk/dayafter-dag.py b/terraform/modules/services/airflow/dags/uk/dayafter-dag.py
@@ -41,7 +41,6 @@
         task_definition='pvlive-national-day-after',
         cluster=cluster,
         overrides={},
-        awslogs_region="eu-west-1",
         launch_type="FARGATE",
         network_configuration={
             "awsvpcConfiguration": {
@@ -52,6 +51,9 @@
         },
         on_failure_callback=on_failure_callback,
         task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/pvlive-national-day-after',
+        awslogs_stream_prefix='streaming/pvlive-national-day-after-consumer',
+        awslogs_region='eu-west-1',
     )
 
 with DAG(
@@ -70,7 +72,6 @@
         cluster=cluster,
         overrides={},
         launch_type="FARGATE",
-        awslogs_group="eu-west-1",
         network_configuration={
             "awsvpcConfiguration": {
                 "subnets": [subnet],
@@ -80,6 +81,9 @@
         },
         on_failure_callback=on_failure_callback,
         task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/pvlive-gsp-day-after',
+        awslogs_stream_prefix='streaming/pvlive-gsp-day-after-consumer',
+        awslogs_region='eu-west-1',
     )
 
     gsp_day_after
@@ -108,6 +112,9 @@
         },
         on_failure_callback=on_failure_callback,
         task_concurrency=10,
+        awslogs_group='/aws/ecs/analysis/metrics',
+        awslogs_stream_prefix='streaming/metrics-analysis',
+        awslogs_region='eu-west-1',
     )
 
     metrics
diff --git a/terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py b/terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py
@@ -47,6 +47,9 @@
         },
         task_concurrency = 10,
         on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/forecast/forecast_pvnet',
+        awslogs_stream_prefix='streaming/forecast_pvnet-forecast',
+        awslogs_region='eu-west-1',
     )
 
     forecast_ecmwf = EcsRunTaskOperator(
@@ -64,7 +67,10 @@
         },
         task_concurrency=10,
         on_failure_callback=on_failure_callback,
-        trigger_rule="all_failed"
+        trigger_rule="all_failed",
+        awslogs_group = '/aws/ecs/forecast/forecast_pvnet_ecmwf',
+        awslogs_stream_prefix = 'streaming/forecast_pvnet_ecmwf-forecast',
+        awslogs_region = 'eu-west-1',
     )
 
     forecast_blend = EcsRunTaskOperator(
@@ -82,7 +88,10 @@
         },
         task_concurrency=10,
         on_failure_callback=on_failure_callback,
-        trigger_rule="one_success"
+        trigger_rule="one_success",
+        awslogs_group='/aws/ecs/blend/forecast_blend',
+        awslogs_stream_prefix='streaming/forecast_blend-blend',
+        awslogs_region='eu-west-1',
     )
 
     latest_only >> forecast >> forecast_blend
@@ -109,6 +118,9 @@
         },
         task_concurrency=10,
         on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/forecast/forecast_pvnet_day_ahead',
+        awslogs_stream_prefix='streaming/forecast_pvnet_day_ahead-forecast',
+        awslogs_region='eu-west-1',
     )
 
     forecast_blend = EcsRunTaskOperator(
@@ -126,6 +138,9 @@
         },
         task_concurrency=10,
         on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/blend/forecast_blend',
+        awslogs_stream_prefix='streaming/forecast_blend-blend',
+        awslogs_region='eu-west-1',
     )
 
     latest_only >> forecast_pvnet_day_ahead >> forecast_blend
diff --git a/terraform/modules/services/airflow/dags/uk/forecast-national-dag.py b/terraform/modules/services/airflow/dags/uk/forecast-national-dag.py
@@ -46,7 +46,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/forecast/forecast_national',
+        awslogs_stream_prefix='streaming/forecast_national-forecast',
+        awslogs_region='eu-west-1',
     )
 
     forecast_blend = EcsRunTaskOperator(
@@ -63,7 +66,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/blend/forecast_blend',
+        awslogs_stream_prefix='streaming/forecast_blend-blend',
+        awslogs_region='eu-west-1',
     )
 
     latest_only >> national_forecast >> forecast_blend
diff --git a/terraform/modules/services/airflow/dags/uk/forecast-site-dag.py b/terraform/modules/services/airflow/dags/uk/forecast-site-dag.py
@@ -46,6 +46,9 @@
         },
         on_failure_callback=on_failure_callback,
      task_concurrency = 10,
+        awslogs_group='/aws/ecs/forecast/pvsite_forecast',
+        awslogs_stream_prefix='streaming/pvsite_forecast-forecast',
+        awslogs_region='eu-west-1',
     )
 
 with DAG(f'{region}-site-forecast-db-clean', schedule_interval="0 0 * * *", default_args=default_args, concurrency=10, max_active_tasks=10) as dag:
@@ -67,7 +70,10 @@
             },
         },
         task_concurrency = 10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/clean/database_clean_up',
+        awslogs_stream_prefix='streaming/database_clean_up-clean',
+        awslogs_region='eu-west-1',
     )
 
 
diff --git a/terraform/modules/services/airflow/dags/uk/nwp-dag.py b/terraform/modules/services/airflow/dags/uk/nwp-dag.py
@@ -52,7 +52,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/consumer/nwp-metoffice',
+        awslogs_stream_prefix='streaming/nwp-metoffice-consumer',
+        awslogs_region='eu-west-1'
     )
 
     nwp_ecmwf_consumer = EcsRunTaskOperator(
@@ -69,6 +72,9 @@
             },
         },
         task_concurrency=10,
+        awslogs_group='/aws/ecs/consumer/nwp-consumer-ecmwf-uk',
+        awslogs_stream_prefix='streaming/nwp-consumer-ecmwf-uk-consumer',
+        awslogs_region='eu-west-1'
     )
 
     rename_zarr_ecmwf = determine_latest_zarr.override(
diff --git a/terraform/modules/services/airflow/dags/uk/pv-dag.py b/terraform/modules/services/airflow/dags/uk/pv-dag.py
@@ -46,7 +46,10 @@
             },
         },
         task_concurrency = 10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/consumer/pv',
+        awslogs_stream_prefix='streaming/pv-consumer',
+        awslogs_region='eu-west-1'
     )
 
     latest_only >> pv_consumer
diff --git a/terraform/modules/services/airflow/dags/uk/pvlive-dag.py b/terraform/modules/services/airflow/dags/uk/pvlive-dag.py
@@ -51,7 +51,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/consumer/pvlive',
+        awslogs_stream_prefix='streaming/pvlive-consumer',
+        awslogs_region='eu-west-1'
     )
 
     command = f'curl -X GET {url}/v0/solar/GB/update_last_data?component=gsp'
diff --git a/terraform/modules/services/airflow/dags/uk/satellite-dag.py b/terraform/modules/services/airflow/dags/uk/satellite-dag.py
@@ -59,7 +59,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/consumer/sat',
+        awslogs_stream_prefix='streaming/sat-consumer',
+        awslogs_region='eu-west-1'
     )
 
     file = f's3://nowcasting-sat-{env}/data/latest/latest.zarr.zip'
@@ -97,7 +100,10 @@
             },
         },
         task_concurrency=10,
-        on_failure_callback=on_failure_callback
+        on_failure_callback=on_failure_callback,
+        awslogs_group='/aws/ecs/consumer/sat-clean-up',
+        awslogs_stream_prefix='streaming/sat-clean-up-consumer',
+        awslogs_region='eu-west-1'
     )
 
     latest_only >> sat_consumer
diff --git a/terraform/modules/services/airflow/iam.tf b/terraform/modules/services/airflow/iam.tf
@@ -82,7 +82,12 @@ resource "aws_iam_policy" "ecs-run" {
           "ecs:RunTask",
           "ecs:DescribeTasks",
           "ecs:StopTask",
-          "iam:PassRole"
+          "iam:PassRole",
+          "logs:GetLogEvents",
+          "logs:GetLogEvents",
+          "logs:GetLogRecord",
+          "logs:GetLogGroupFields",
+          "logs:GetQueryResults"
         ]
         Effect   = "Allow"
         Resource = "*"
@@ -154,7 +159,7 @@ resource "aws_iam_policy" "elb-auto-scaling"{
 				"elasticloadbalancing:DescribeLoadBalancers",
 				"logs:CreateLogGroup",
 				"logs:PutRetentionPolicy",
-				"logs:DescribeLogGroups"
+				"logs:DescribeLogGroups",
 			],
 			"Resource": "*"
 		}

Original file line number	Diff line number	Diff line change
`@@ -50,6 +50,9 @@`
`50`	`50`	`},`
`51`	`51`	`on_failure_callback=on_failure_callback,`
`52`	`52`	`task_concurrency=10,`
	`53`	`+ awslogs_group='/aws/ecs/consumer/runvl-consumer',`
	`54`	`+ awslogs_stream_prefix='streaming/runvl-consumer-consumer',`
	`55`	`+ awslogs_region='ap-south-1'`
`53`	`56`	`)`
`54`	`57`
`55`	`58`