diff --git a/backend/pycon/tasks.py b/backend/pycon/tasks.py index 8e8750a79c..4b8d10bc36 100644 --- a/backend/pycon/tasks.py +++ b/backend/pycon/tasks.py @@ -27,10 +27,14 @@ def launch_heavy_processing_worker(): if settings.ENVIRONMENT == "local": return - cluster_name = f"pythonit-{settings.ENVIRONMENT}-heavy-processing-worker" + cluster_name = f"pythonit-{settings.ENVIRONMENT}" ecs_client = boto3.client("ecs", region_name=settings.AWS_REGION_NAME) - response = ecs_client.list_tasks(cluster=cluster_name, desiredStatus="RUNNING") + response = ecs_client.list_tasks( + cluster=cluster_name, + desiredStatus="RUNNING", + family=f"pythonit-{settings.ENVIRONMENT}-heavy-processing-worker", + ) if len(response["taskArns"]) > 0: return diff --git a/backend/pycon/tests/test_tasks.py b/backend/pycon/tests/test_tasks.py index ef8eb4d6aa..13a7e68e50 100644 --- a/backend/pycon/tests/test_tasks.py +++ b/backend/pycon/tests/test_tasks.py @@ -39,15 +39,17 @@ def test_launch_heavy_processing_worker_starts_task(settings, mocker): launch_heavy_processing_worker() mock_client.return_value.describe_tasks.assert_called_with( - cluster="pythonit-production-heavy-processing-worker", tasks=["arn-abc"] + cluster="pythonit-production", tasks=["arn-abc"] ) mock_client.return_value.list_tasks.assert_called_with( - cluster="pythonit-production-heavy-processing-worker", desiredStatus="RUNNING" + cluster="pythonit-production", + desiredStatus="RUNNING", + family="pythonit-production-heavy-processing-worker", ) mock_client.return_value.run_task.assert_called_with( - cluster="pythonit-production-heavy-processing-worker", + cluster="pythonit-production", taskDefinition="pythonit-production-heavy-processing-worker", count=1, networkConfiguration={ diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf index 981c35e9a0..3da3052687 100644 --- a/infrastructure/applications/applications.tf +++ b/infrastructure/applications/applications.tf @@ -26,6 +26,7 @@ module "pycon_backend" { security_group_id = module.cluster.security_group_id server_ip = module.cluster.server_ip logs_group_name = module.cluster.logs_group_name + iam_role_arn = module.cluster.iam_role_arn providers = { aws = aws diff --git a/infrastructure/applications/cluster/iam.tf b/infrastructure/applications/cluster/iam.tf index e1a98f3e2a..3eb193828b 100644 --- a/infrastructure/applications/cluster/iam.tf +++ b/infrastructure/applications/cluster/iam.tf @@ -20,7 +20,7 @@ data "aws_iam_policy_document" "server_assume_role" { principals { type = "Service" - identifiers = ["ec2.amazonaws.com", "ecs-tasks.amazonaws.com"] + identifiers = ["ec2.amazonaws.com", "ecs-tasks.amazonaws.com", "ecs.amazonaws.com"] } actions = ["sts:AssumeRole"] @@ -35,7 +35,6 @@ data "aws_iam_policy_document" "server_role_policy" { "ses:*", "ecs:*", "ecr:*", - "ec2:DescribeInstances", ] resources = [ "*" @@ -78,4 +77,34 @@ data "aws_iam_policy_document" "server_role_policy" { resources = ["*"] effect = "Allow" } + + statement { + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:CreateVolume", + "ec2:AttachVolume", + "ec2:DetachVolume", + "ec2:CreateTags", + "ec2:DeleteVolume", + "ec2:DescribeVolumes", + ] + resources = ["*"] + effect = "Allow" + } + + statement { + actions = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ] + resources = ["*"] + effect = "Allow" + } +} + +output "iam_role_arn" { + value = aws_iam_role.server.arn } diff --git a/infrastructure/applications/cluster/security.tf b/infrastructure/applications/cluster/security.tf index cdceb0c581..d0ec4ece7d 100644 --- a/infrastructure/applications/cluster/security.tf +++ b/infrastructure/applications/cluster/security.tf @@ -22,6 +22,25 @@ resource "aws_security_group_rule" "server_rds" { security_group_id = aws_security_group.server.id } +resource "aws_security_group_rule" "in_redis" { + type = "egress" + from_port = 6379 + to_port = 6379 + protocol = "tcp" + source_security_group_id = aws_security_group.server.id + security_group_id = aws_security_group.server.id +} + +resource "aws_security_group_rule" "out_redis" { + # needed by fargate to connect to the server with redis + type = "ingress" + from_port = 6379 + to_port = 6379 + protocol = "tcp" + source_security_group_id = aws_security_group.server.id + security_group_id = aws_security_group.server.id +} + resource "aws_security_group_rule" "web_http" { type = "ingress" from_port = 80 diff --git a/infrastructure/applications/pycon_backend/variables.tf b/infrastructure/applications/pycon_backend/variables.tf index 582eecfad8..23f0956687 100644 --- a/infrastructure/applications/pycon_backend/variables.tf +++ b/infrastructure/applications/pycon_backend/variables.tf @@ -9,3 +9,4 @@ variable "cluster_id" {} variable "security_group_id" {} variable "server_ip" {} variable "logs_group_name" {} +variable "iam_role_arn" {} diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/worker.tf index ca053d434d..7cd0ee725e 100644 --- a/infrastructure/applications/pycon_backend/worker.tf +++ b/infrastructure/applications/pycon_backend/worker.tf @@ -169,15 +169,13 @@ locals { value = jsonencode({ subnets = [data.aws_subnet.public_1a.id], security_groups = [ - data.aws_security_group.rds.id, - data.aws_security_group.lambda.id, - aws_security_group.instance.id + var.security_group_id ], }) }, { name = "ECS_SERVICE_ROLE", - value = aws_iam_role.ecs_service.arn + value = var.iam_role_arn }, { name = "AWS_SES_CONFIGURATION_SET" diff --git a/infrastructure/applications/pycon_backend/worker_heavy_processing.tf b/infrastructure/applications/pycon_backend/worker_heavy_processing.tf index 5f1561219f..84a539a8fa 100644 --- a/infrastructure/applications/pycon_backend/worker_heavy_processing.tf +++ b/infrastructure/applications/pycon_backend/worker_heavy_processing.tf @@ -1,33 +1,21 @@ -resource "aws_ecs_cluster" "heavy_processing_worker" { - name = "pythonit-${terraform.workspace}-heavy-processing-worker" - - setting { - name = "containerInsights" - value = "enabled" - } -} - -resource "aws_cloudwatch_log_group" "heavy_processing_worker_logs" { - name = "/ecs/pythonit-${terraform.workspace}-heavy-processing-worker" - retention_in_days = 7 -} - resource "aws_ecs_task_definition" "heavy_processing_worker" { family = "pythonit-${terraform.workspace}-heavy-processing-worker" requires_compatibilities = ["FARGATE"] cpu = 4096 memory = 16384 network_mode = "awsvpc" - execution_role_arn = aws_iam_role.worker.arn - task_role_arn = aws_iam_role.worker.arn + execution_role_arn = var.iam_role_arn + task_role_arn = var.iam_role_arn ephemeral_storage { size_in_gib = 21 } + runtime_platform { operating_system_family = "LINUX" cpu_architecture = "ARM64" } + container_definitions = jsonencode([ { name = "worker" @@ -62,9 +50,9 @@ resource "aws_ecs_task_definition" "heavy_processing_worker" { logConfiguration = { logDriver = "awslogs" options = { - "awslogs-group" = aws_cloudwatch_log_group.heavy_processing_worker_logs.name + "awslogs-group" = var.logs_group_name "awslogs-region" = "eu-central-1" - "awslogs-stream-prefix" = "ecs" + "awslogs-stream-prefix" = "heavy-processing-worker" } } diff --git a/infrastructure/applications/pycon_backend/worker_role.tf b/infrastructure/applications/pycon_backend/worker_role.tf deleted file mode 100644 index b1666e4125..0000000000 --- a/infrastructure/applications/pycon_backend/worker_role.tf +++ /dev/null @@ -1,111 +0,0 @@ -resource "aws_iam_role" "worker" { - name = "pythonit-${terraform.workspace}-worker" - - assume_role_policy = <