update ecmwf to new nwp-consumer (#734)

* update ecmwf to new nwp-consumer

* update

Author: peterdudfield

terraform/nowcasting/production/main.tf

@@ -191,44 +191,51 @@ module "nwp-metoffice" {
 
 # 3.3
 module "nwp-ecmwf" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=26e3b29"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=2b68542"
 
-  ecs-task_name = "nwp-consumer-ecmwf-uk"
-  ecs-task_type = "consumer"
+  ecs-task_name               = "nwp-consumer-ecmwf-uk"
+  ecs-task_type               = "consumer"
   ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn
 
-  aws-region                     = var.region
-  aws-environment                = local.environment
-  aws-secretsmanager_secret_arn = aws_secretsmanager_secret.nwp_consumer_secret.arn
+  ecs-task_size = {
+    cpu = 512
+    memory = 1024
+  }
 
-  s3-buckets = [
-    {
-      id : module.s3.s3-nwp-bucket.id
-      access_policy_arn : module.s3.iam-policy-s3-nwp-write.arn
-    }
-  ]
+  aws-region                    = var.region
+  aws-environment               = local.environment
+
+  s3-buckets = [{
+    id : module.s3.s3-nwp-bucket.id
+    access_policy_arn : module.s3.iam-policy-s3-nwp-write.arn
+  }]
 
   container-env_vars = [
+    { "name" : "MODEL_REPOSITORY", "value" : "ecmwf-realtime" },
     { "name" : "AWS_REGION", "value" : "eu-west-1" },
-    { "name" : "AWS_S3_BUCKET", "value" : module.s3.s3-nwp-bucket.id },
+    { "name" : "ECMWF_REALTIME_S3_REGION", "value": "eu-west-1" },
+    { "name" : "ECMWF_REALTIME_S3_BUCKET", "value" : "ocf-ecmwf-production" },
+    { "name" : "ZARRDIR", "value" : "s3://${module.s3.s3-nwp-bucket.id}/ecmwf/data" },
     { "name" : "LOGLEVEL", "value" : "DEBUG" },
+    { "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
+    { "name" : "CONCURRENCY", "value" : "false" },
+    # legacy ones
+    { "name" : "AWS_S3_BUCKET", "value" : module.s3.s3-nwp-bucket.id },
     { "name" : "ECMWF_AWS_REGION", "value": "eu-west-1" },
     { "name" : "ECMWF_AWS_S3_BUCKET", "value" : "ocf-ecmwf-production" },
     { "name" : "ECMWF_AREA", "value" : "uk" },
-    { "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
     { "name" : "ENVIRONMENT", "value" : local.environment },
+    { "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
+    { "name" : "LOGLEVEL", "value" : "DEBUG" }
   ]
-  container-secret_vars = ["ECMWF_AWS_ACCESS_KEY", "ECMWF_AWS_ACCESS_SECRET"]
-  container-tag         = var.nwp_version
-  container-name        = "openclimatefix/nwp-consumer"
-  container-command     = [
-    "download",
-    "--source=ecmwf-s3",
-    "--sink=s3",
-    "--rdir=ecmwf/raw",
-    "--zdir=ecmwf/data",
-    "--create-latest"
+
+    container-secret_vars = [
+  {secret_policy_arn: aws_secretsmanager_secret.nwp_consumer_secret.arn,
+  values: ["ECMWF_REALTIME_S3_ACCESS_KEY", "ECMWF_REALTIME_S3_ACCESS_SECRET"]}
   ]
+    container-tag         = var.nwp_ecmwf_version
+  container-name        = "openclimatefix/nwp-consumer"
+  container-command     = ["consume"]
 }
 
 
@@ -681,7 +688,7 @@ module "forecast_blend" {
 
 # 5.2
 module "airflow" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=4d6eadd"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=2b68542"
 
   aws-environment   = local.environment
   aws-domain        = local.domain

terraform/nowcasting/production/variables.tf

@@ -18,6 +18,11 @@ variable "nwp_version" {
   description = "The NWP version"
 }
 
+variable "nwp_ecmwf_version" {
+  description = "The ECMWF NWP version"
+  default="1.0.9"
+}
+
 variable "sat_version" {
   description = "The Satellite version"
 }