Run heavy processing tasks in the same server cluster (#4174)

Author: marcoacierno

backend/pycon/tasks.py

@@ -27,10 +27,14 @@ def launch_heavy_processing_worker():
     if settings.ENVIRONMENT == "local":
         return
 
-    cluster_name = f"pythonit-{settings.ENVIRONMENT}-heavy-processing-worker"
+    cluster_name = f"pythonit-{settings.ENVIRONMENT}"
     ecs_client = boto3.client("ecs", region_name=settings.AWS_REGION_NAME)
 
-    response = ecs_client.list_tasks(cluster=cluster_name, desiredStatus="RUNNING")
+    response = ecs_client.list_tasks(
+        cluster=cluster_name,
+        desiredStatus="RUNNING",
+        family=f"pythonit-{settings.ENVIRONMENT}-heavy-processing-worker",
+    )
 
     if len(response["taskArns"]) > 0:
         return

backend/pycon/tests/test_tasks.py

@@ -39,15 +39,17 @@ def test_launch_heavy_processing_worker_starts_task(settings, mocker):
     launch_heavy_processing_worker()
 
     mock_client.return_value.describe_tasks.assert_called_with(
-        cluster="pythonit-production-heavy-processing-worker", tasks=["arn-abc"]
+        cluster="pythonit-production", tasks=["arn-abc"]
     )
 
     mock_client.return_value.list_tasks.assert_called_with(
-        cluster="pythonit-production-heavy-processing-worker", desiredStatus="RUNNING"
+        cluster="pythonit-production",
+        desiredStatus="RUNNING",
+        family="pythonit-production-heavy-processing-worker",
     )
 
     mock_client.return_value.run_task.assert_called_with(
-        cluster="pythonit-production-heavy-processing-worker",
+        cluster="pythonit-production",
         taskDefinition="pythonit-production-heavy-processing-worker",
         count=1,
         networkConfiguration={

infrastructure/applications/applications.tf

@@ -26,6 +26,7 @@ module "pycon_backend" {
   security_group_id = module.cluster.security_group_id
   server_ip = module.cluster.server_ip
   logs_group_name = module.cluster.logs_group_name
+  iam_role_arn = module.cluster.iam_role_arn
 
   providers = {
     aws    = aws

infrastructure/applications/cluster/iam.tf

@@ -20,7 +20,7 @@ data "aws_iam_policy_document" "server_assume_role" {
 
     principals {
       type        = "Service"
-      identifiers = ["ec2.amazonaws.com", "ecs-tasks.amazonaws.com"]
+      identifiers = ["ec2.amazonaws.com", "ecs-tasks.amazonaws.com", "ecs.amazonaws.com"]
     }
 
     actions = ["sts:AssumeRole"]
@@ -35,7 +35,6 @@ data "aws_iam_policy_document" "server_role_policy" {
       "ses:*",
       "ecs:*",
       "ecr:*",
-      "ec2:DescribeInstances",
     ]
     resources = [
       "*"
@@ -78,4 +77,34 @@ data "aws_iam_policy_document" "server_role_policy" {
     resources = ["*"]
     effect    = "Allow"
   }
+
+  statement {
+    actions = [
+      "ec2:DescribeAvailabilityZones",
+      "ec2:DescribeInstances",
+      "ec2:CreateVolume",
+      "ec2:AttachVolume",
+      "ec2:DetachVolume",
+      "ec2:CreateTags",
+      "ec2:DeleteVolume",
+      "ec2:DescribeVolumes",
+    ]
+    resources = ["*"]
+    effect    = "Allow"
+  }
+
+  statement {
+    actions = [
+      "ssmmessages:CreateControlChannel",
+      "ssmmessages:CreateDataChannel",
+      "ssmmessages:OpenControlChannel",
+      "ssmmessages:OpenDataChannel"
+    ]
+    resources = ["*"]
+    effect    = "Allow"
+  }
+}
+
+output "iam_role_arn" {
+  value = aws_iam_role.server.arn
 }

infrastructure/applications/cluster/security.tf

@@ -22,6 +22,25 @@ resource "aws_security_group_rule" "server_rds" {
   security_group_id = aws_security_group.server.id
 }
 
+resource "aws_security_group_rule" "in_redis" {
+  type              = "egress"
+  from_port         = 6379
+  to_port           = 6379
+  protocol          = "tcp"
+  source_security_group_id = aws_security_group.server.id
+  security_group_id = aws_security_group.server.id
+}
+
+resource "aws_security_group_rule" "out_redis" {
+  # needed by fargate to connect to the server with redis
+  type              = "ingress"
+  from_port         = 6379
+  to_port           = 6379
+  protocol          = "tcp"
+  source_security_group_id = aws_security_group.server.id
+  security_group_id = aws_security_group.server.id
+}
+
 resource "aws_security_group_rule" "web_http" {
   type              = "ingress"
   from_port         = 80

infrastructure/applications/pycon_backend/variables.tf

@@ -9,3 +9,4 @@ variable "cluster_id" {}
 variable "security_group_id" {}
 variable "server_ip" {}
 variable "logs_group_name" {}
+variable "iam_role_arn" {}

infrastructure/applications/pycon_backend/worker.tf

@@ -169,15 +169,13 @@ locals {
       value = jsonencode({
         subnets = [data.aws_subnet.public_1a.id],
         security_groups = [
-          data.aws_security_group.rds.id,
-          data.aws_security_group.lambda.id,
-          aws_security_group.instance.id
+          var.security_group_id
         ],
       })
     },
     {
       name = "ECS_SERVICE_ROLE",
-      value = aws_iam_role.ecs_service.arn
+      value = var.iam_role_arn
     },
     {
       name = "AWS_SES_CONFIGURATION_SET"

infrastructure/applications/pycon_backend/worker_heavy_processing.tf

@@ -1,33 +1,21 @@
-resource "aws_ecs_cluster" "heavy_processing_worker" {
-  name = "pythonit-${terraform.workspace}-heavy-processing-worker"
-
-  setting {
-    name  = "containerInsights"
-    value = "enabled"
-  }
-}
-
-resource "aws_cloudwatch_log_group" "heavy_processing_worker_logs" {
-  name              = "/ecs/pythonit-${terraform.workspace}-heavy-processing-worker"
-  retention_in_days = 7
-}
-
 resource "aws_ecs_task_definition" "heavy_processing_worker" {
   family = "pythonit-${terraform.workspace}-heavy-processing-worker"
   requires_compatibilities = ["FARGATE"]
   cpu                      = 4096
   memory                   = 16384
   network_mode             = "awsvpc"
-  execution_role_arn = aws_iam_role.worker.arn
-  task_role_arn = aws_iam_role.worker.arn
+  execution_role_arn = var.iam_role_arn
+  task_role_arn = var.iam_role_arn
 
   ephemeral_storage {
     size_in_gib = 21
   }
+
   runtime_platform {
     operating_system_family = "LINUX"
     cpu_architecture = "ARM64"
   }
+
   container_definitions = jsonencode([
     {
       name              = "worker"
@@ -62,9 +50,9 @@ resource "aws_ecs_task_definition" "heavy_processing_worker" {
       logConfiguration = {
         logDriver = "awslogs"
         options = {
-          "awslogs-group"         = aws_cloudwatch_log_group.heavy_processing_worker_logs.name
+          "awslogs-group"         = var.logs_group_name
           "awslogs-region"        = "eu-central-1"
-          "awslogs-stream-prefix" = "ecs"
+          "awslogs-stream-prefix" = "heavy-processing-worker"
         }
       }