add ecs task and dag for neso foreacst

Author: peterdudfield

terraform/modules/services/airflow/dags/uk/forecast-national-dag.py

@@ -79,3 +79,39 @@
     )
 
     latest_only >> national_forecast >> forecast_blend
+
+
+
+with DAG(
+    f"{region}-neso-forecast",
+    schedule_interval="0 * * * *",
+    default_args=default_args,
+    concurrency=10,
+    max_active_tasks=10,
+) as dag:
+    dag.doc_md = "Get PV data"
+
+    latest_only = LatestOnlyOperator(task_id="latest_only")
+
+    neso_forecast = EcsRunTaskOperator(
+        task_id=f"{region}-neso-forecast",
+        task_definition="neso-forecast",
+        cluster=cluster,
+        overrides={},
+        launch_type="FARGATE",
+        network_configuration={
+            "awsvpcConfiguration": {
+                "subnets": [subnet],
+                "securityGroups": [security_group],
+                "assignPublicIp": "ENABLED",
+            },
+        },
+        task_concurrency=10,
+        on_failure_callback=on_failure_callback,
+        awslogs_group="/aws/ecs/consume/neso-forecast",
+        awslogs_stream_prefix="streaming/neso-forecast-consume",
+        awslogs_region="eu-west-1",
+    )
+
+
+    latest_only >> neso_forecast

terraform/nowcasting/development/main.tf

@@ -21,6 +21,7 @@ The componentes ares:
 3.7 - PVLive Consumer (From PVLive)
 3.8 - PVLive Consumer - GSP Day After
 3.9 - PVLive Consumer - National Day After
+3.10 - NESO Forecast Consumer
 4.1 - Metrics
 4.2 - Forecast PVnet 1
 4.3 - Forecast National XG
@@ -472,6 +473,37 @@ module "gsp-consumer-day-after-national" {
   container-command     = []
 }
 
+# 3.10
+module "neso-forecast-consumer" {
+  source = "../../modules/services/ecs_task"
+
+  ecs-task_name = "neso-forecast"
+  ecs-task_type = "consumer"
+  ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn
+  ecs-task_size = {
+    cpu    = 256
+    memory = 512
+  }
+
+  aws-region                     = var.region
+  aws-environment                = local.environment
+
+  s3-buckets = []
+
+  container-env_vars = [
+    { "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
+    { "name" : "ENVIRONMENT", "value" : local.environment },
+  ]
+  container-secret_vars = [
+  {secret_policy_arn: module.database.forecast-database-secret.arn,
+  values: ["DB_URL"]}
+  ]
+  container-tag         = var.neso_forecast_consumer_version
+  container-name        = "openclimatefix/neso_solar_consumer_api"
+  container-registry = "docker.io"
+  container-command     = []
+}
+
 # 4.1
 module "metrics" {
   source = "../../modules/services/ecs_task"

terraform/nowcasting/development/variables.tf

@@ -123,3 +123,9 @@ variable "sentry_dsn_api" {
   description = "The Sentry DSN for all backend components"
   default     = ""
 }
+
+variable "neso_forecast_consumer_version" {
+  type        = string
+  description = "The version of the neso forecast consumer"
+  default     = "1.0.2"
+}