Run heavy processing tasks in the same server cluster

Author: marcoacierno

backend/pycon/tasks.py

@@ -27,7 +27,7 @@ def launch_heavy_processing_worker():
     if settings.ENVIRONMENT == "local":
         return
 
-    cluster_name = f"pythonit-{settings.ENVIRONMENT}-heavy-processing-worker"
+    cluster_name = f"pythonit-{settings.ENVIRONMENT}"
     ecs_client = boto3.client("ecs", region_name=settings.AWS_REGION_NAME)
 
     response = ecs_client.list_tasks(cluster=cluster_name, desiredStatus="RUNNING")

infrastructure/applications/applications.tf

@@ -26,6 +26,7 @@ module "pycon_backend" {
   security_group_id = module.cluster.security_group_id
   server_ip = module.cluster.server_ip
   logs_group_name = module.cluster.logs_group_name
+  iam_role_arn = module.cluster.iam_role_arn
 
   providers = {
     aws    = aws

infrastructure/applications/cluster/iam.tf

@@ -79,3 +79,7 @@ data "aws_iam_policy_document" "server_role_policy" {
     effect    = "Allow"
   }
 }
+
+output "iam_role_arn" {
+  value = aws_iam_role.server.arn
+}

infrastructure/applications/pycon_backend/variables.tf

@@ -9,3 +9,4 @@ variable "cluster_id" {}
 variable "security_group_id" {}
 variable "server_ip" {}
 variable "logs_group_name" {}
+variable "iam_role_arn" {}

infrastructure/applications/pycon_backend/worker_heavy_processing.tf

@@ -1,25 +1,11 @@
-resource "aws_ecs_cluster" "heavy_processing_worker" {
-  name = "pythonit-${terraform.workspace}-heavy-processing-worker"
-
-  setting {
-    name  = "containerInsights"
-    value = "enabled"
-  }
-}
-
-resource "aws_cloudwatch_log_group" "heavy_processing_worker_logs" {
-  name              = "/ecs/pythonit-${terraform.workspace}-heavy-processing-worker"
-  retention_in_days = 7
-}
-
 resource "aws_ecs_task_definition" "heavy_processing_worker" {
   family = "pythonit-${terraform.workspace}-heavy-processing-worker"
   requires_compatibilities = ["FARGATE"]
   cpu                      = 4096
   memory                   = 16384
   network_mode             = "awsvpc"
-  execution_role_arn = aws_iam_role.worker.arn
-  task_role_arn = aws_iam_role.worker.arn
+  execution_role_arn = var.iam_role_arn
+  task_role_arn = var.iam_role_arn
 
   ephemeral_storage {
     size_in_gib = 21
@@ -62,9 +48,9 @@ resource "aws_ecs_task_definition" "heavy_processing_worker" {
       logConfiguration = {
         logDriver = "awslogs"
         options = {
-          "awslogs-group"         = aws_cloudwatch_log_group.heavy_processing_worker_logs.name
+          "awslogs-group"         = var.logs_group_name
           "awslogs-region"        = "eu-central-1"
-          "awslogs-stream-prefix" = "ecs"
+          "awslogs-stream-prefix" = "heavy-processing-worker"
         }
       }