Improve ecs Module

infrastructure/README.md

@@ -21,7 +21,8 @@ Follow these steps to set up the infrastructure:
   ```bash
   cd infrastructure/backend/
   ```
-*Note:* Optionally change the region: set `aws_region` in a `.tfvars` file.
+
+**Note:** Optionally change the region: set `aws_region` in a `.tfvars` file.
 
 - Initialize Terraform if needed:
   ```bash
@@ -33,6 +34,10 @@ Follow these steps to set up the infrastructure:
   terraform apply
   ```
 
+**Note:** Copy the state bucket name from the output.
+
+**Note:** It is recommended to not destroy the backend resources unless absolutely necessary.
+
 2. **Setup Main Infrastructure (staging)**:
 
 - Navigate to the main infrastructure directory. If you are in `infrastructure/backend`, you can use:
@@ -50,13 +55,23 @@ Follow these steps to set up the infrastructure:
   cat terraform.tfvars.example > terraform.tfvars
   ```
 
-- *Note:* Optionally change the region:
-  - set `aws_region` in a `.tfvars` file.
-  - set `region` in a `.tfbackend` file and provide it using `terraform init -backend-config=<file>`.
+- Create a local backend configuration file:
+  ```bash
+  touch terraform.tfbackend
+  ```
+
+- Copy the contents from the example file:
+  ```bash
+  cat terraform.tfbackend.example > terraform.tfbackend
+  ```
+
+*Note:* Update the state bucket name in `terraform.tfbackend` with the name of the state bucket created in the previous step.
+
+*Note:* Update defaults (e.g. `region`) as needed.
 
 - Initialize Terraform with the backend configuration:
   ```bash
-  terraform init
+  terraform init -backend-config=terraform.tfbackend
   ```
 
 - Apply the changes to create the main infrastructure using the command:
@@ -114,13 +129,15 @@ The Django backend deployment is managed by Zappa. This includes the API Gateway
 
 5. **Deploy**:
 
-    - *Note*: Make sure to populate all `DJANGO_*` secrets that are set as `to-be-set-in-aws-console`
+    - **Note**: Make sure to populate all `DJANGO_*` secrets that are set as `to-be-set-in-aws-console`
       in the Parameter Store. The deployment might fail with no logs if secrets such as
       `DJANGO_SLACK_BOT_TOKEN` are invalid.
 
     ```bash
     zappa deploy staging
     ```
+    - **Note**: If the deployment is successful but returns a `5xx` error, resolve the issues
+      and use `zappa undeploy staging` & `zappa deploy staging`. The command `zappa update staging` may not work.
 
 Once deployed, use the URL provided by Zappa to test the API.
 
@@ -163,7 +180,7 @@ Migrate and load data into the new database.
    - Upload the fixture present in `backend/data` to `nest-fixtures` bucket using the following command:
 
      ```bash
-     aws s3 cp data/nest.json.gz s3://nest-fixtures/
+     aws s3 cp data/nest.json.gz s3://owasp-nest-fixtures-<id>/
      ```
 
 3. **Run ECS Tasks**:
@@ -188,6 +205,9 @@ Migrate and load data into the new database.
   zappa undeploy staging
   ```
 
+- Ensure all buckets and ECR repositories are empty.
+
+**Note:** Some resources have `prevent_destroy` set to `true`. Please set it to `false` before destruction.
 - To destroy Terraform infrastructure:
 
   ```bash

infrastructure/backend/main.tf

@@ -5,6 +5,10 @@ terraform {
       source  = "hashicorp/aws"
       version = "6.22.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.7.2"
+    }
   }
 }
 
@@ -46,6 +50,10 @@ data "aws_iam_policy_document" "state_https_only" {
   }
 }
 
+resource "random_id" "suffix" {
+  byte_length = 4
+}
+
 resource "aws_dynamodb_table" "state_lock" {
   name         = "${var.project_name}-terraform-state-lock"
   billing_mode = "PAY_PER_REQUEST"
@@ -58,20 +66,32 @@ resource "aws_dynamodb_table" "state_lock" {
     name = "LockID"
     type = "S"
   }
+  lifecycle {
+    prevent_destroy = true
+  }
   point_in_time_recovery {
     enabled = true
   }
 }
 
 resource "aws_s3_bucket" "logs" { # NOSONAR
-  bucket = "${var.project_name}-terraform-state-logs"
+  bucket = "${var.project_name}-terraform-state-logs-${random_id.suffix.hex}"
+
+  lifecycle {
+    prevent_destroy = true
+  }
   tags = {
     Name = "${var.project_name}-terraform-state-logs"
   }
 }
 
 resource "aws_s3_bucket" "state" { # NOSONAR
-  bucket = "${var.project_name}-terraform-state"
+  bucket              = "${var.project_name}-terraform-state-${random_id.suffix.hex}"
+  object_lock_enabled = true
+
+  lifecycle {
+    prevent_destroy = true
+  }
   tags = {
     Name = "${var.project_name}-terraform-state"
   }
@@ -115,6 +135,17 @@ resource "aws_s3_bucket_logging" "state" {
   target_prefix = "s3/"
 }
 
+resource "aws_s3_bucket_object_lock_configuration" "state" {
+  bucket = aws_s3_bucket.state.id
+
+  rule {
+    default_retention {
+      days = 30
+      mode = "GOVERNANCE"
+    }
+  }
+}
+
 resource "aws_s3_bucket_policy" "logs" {
   bucket = aws_s3_bucket.logs.id
   policy = data.aws_iam_policy_document.logs.json

infrastructure/modules/ecs/main.tf

@@ -16,6 +16,28 @@ resource "aws_ecs_cluster" "main" {
   tags = var.common_tags
 }
 
+resource "aws_ecr_lifecycle_policy" "main" {
+  repository = aws_ecr_repository.main.name
+
+  policy = jsonencode({
+    rules = [
+      {
+        rulePriority = 1
+        description  = "Remove untagged images"
+        selection = {
+          tagStatus   = "untagged"
+          countType   = "sinceImagePushed"
+          countUnit   = "days"
+          countNumber = 7
+        }
+        action = {
+          type = "expire"
+        }
+      }
+    ]
+  })
+}
+
 resource "aws_ecr_repository" "main" {
   name = "${var.project_name}-${var.environment}-backend"
   image_scanning_configuration {
@@ -60,8 +82,43 @@ resource "aws_iam_policy" "ecs_tasks_execution_role_ssm_policy" {
   })
 }
 
-resource "aws_iam_role_policy_attachment" "ecs_tasks_execution_role_policy" {
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+resource "aws_iam_policy" "ecs_tasks_execution_policy" {
+  name        = "${var.project_name}-${var.environment}-ecs-tasks-execution-policy"
+  description = "Custom policy for ECS task execution - ECR and CloudWatch Logs access"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "ecr:GetAuthorizationToken"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      },
+      {
+        Action = [
+          "ecr:BatchCheckLayerAvailability",
+          "ecr:BatchGetImage",
+          "ecr:GetDownloadUrlForLayer"
+        ]
+        Effect   = "Allow"
+        Resource = aws_ecr_repository.main.arn
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "logs:CreateLogStream",
+          "logs:PutLogEvents"
+        ]
+        Resource = "arn:aws:logs:${var.aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/ecs/${var.project_name}-${var.environment}-*:*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "ecs_tasks_execution_policy_attachment" {
+  policy_arn = aws_iam_policy.ecs_tasks_execution_policy.arn
   role       = aws_iam_role.ecs_tasks_execution_role.name
 }
 
@@ -111,8 +168,37 @@ resource "aws_iam_role" "event_bridge_role" {
   })
 }
 
-resource "aws_iam_role_policy_attachment" "event_bridge_role_policy" {
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole"
+resource "aws_iam_policy" "event_bridge_ecs_policy" {
+  name        = "${var.project_name}-${var.environment}-event-bridge-ecs-policy"
+  description = "Allow EventBridge to run ECS tasks"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "ecs:RunTask"
+        Effect = "Allow"
+        Condition = {
+          ArnLike = {
+            "ecs:cluster" = aws_ecs_cluster.main.arn
+          }
+        }
+        Resource = "arn:aws:ecs:${var.aws_region}:${data.aws_caller_identity.current.account_id}:task-definition/${var.project_name}-${var.environment}-*:*"
+      },
+      {
+        Action = "iam:PassRole"
+        Effect = "Allow"
+        Resource = [
+          aws_iam_role.ecs_task_role.arn,
+          aws_iam_role.ecs_tasks_execution_role.arn
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "event_bridge_policy_attachment" {
+  policy_arn = aws_iam_policy.event_bridge_ecs_policy.arn
   role       = aws_iam_role.event_bridge_role.name
 }
 
@@ -128,7 +214,7 @@ module "sync_data_task" {
   ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn
   environment                  = var.environment
   event_bridge_role_arn        = aws_iam_role.event_bridge_role.arn
-  image_url                    = aws_ecr_repository.main.repository_url
+  image_url                    = "${aws_ecr_repository.main.repository_url}:${var.image_tag}"
   memory                       = var.sync_data_task_memory
   private_subnet_ids           = var.private_subnet_ids
   project_name                 = var.project_name
@@ -157,7 +243,7 @@ module "owasp_update_project_health_metrics_task" {
   ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn
   environment                  = var.environment
   event_bridge_role_arn        = aws_iam_role.event_bridge_role.arn
-  image_url                    = aws_ecr_repository.main.repository_url
+  image_url                    = "${aws_ecr_repository.main.repository_url}:${var.image_tag}"
   memory                       = var.update_project_health_metrics_task_memory
   private_subnet_ids           = var.private_subnet_ids
   project_name                 = var.project_name
@@ -178,7 +264,7 @@ module "owasp_update_project_health_scores_task" {
   ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn
   environment                  = var.environment
   event_bridge_role_arn        = aws_iam_role.event_bridge_role.arn
-  image_url                    = aws_ecr_repository.main.repository_url
+  image_url                    = "${aws_ecr_repository.main.repository_url}:${var.image_tag}"
   memory                       = var.update_project_health_scores_task_memory
   private_subnet_ids           = var.private_subnet_ids
   project_name                 = var.project_name
@@ -198,7 +284,7 @@ module "migrate_task" {
   ecs_cluster_arn              = aws_ecs_cluster.main.arn
   ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn
   environment                  = var.environment
-  image_url                    = "${aws_ecr_repository.main.repository_url}:latest"
+  image_url                    = "${aws_ecr_repository.main.repository_url}:${var.image_tag}"
   memory                       = var.migrate_task_memory
   private_subnet_ids           = var.private_subnet_ids
   project_name                 = var.project_name
@@ -215,9 +301,11 @@ module "load_data_task" {
     "-c",
     <<-EOT
     set -e
-    pip install --target=/tmp/awscli-packages awscli
-    export PYTHONPATH="/tmp/awscli-packages:$PYTHONPATH"
-    python /tmp/awscli-packages/bin/aws s3 cp s3://${var.fixtures_s3_bucket}/nest.json.gz /tmp/nest.json.gz
+    python -c "
+    import boto3
+    s3 = boto3.client('s3')
+    s3.download_file('${var.fixtures_bucket_name}', 'nest.json.gz', '/tmp/nest.json.gz')
+    "
     python manage.py load_data --fixture-path /tmp/nest.json.gz
     EOT
   ]
@@ -227,7 +315,7 @@ module "load_data_task" {
   ecs_cluster_arn              = aws_ecs_cluster.main.arn
   ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn
   environment                  = var.environment
-  image_url                    = "${aws_ecr_repository.main.repository_url}:latest"
+  image_url                    = "${aws_ecr_repository.main.repository_url}:${var.image_tag}"
   memory                       = var.load_data_task_memory
   private_subnet_ids           = var.private_subnet_ids
   project_name                 = var.project_name
@@ -247,7 +335,7 @@ module "index_data_task" {
   ecs_cluster_arn              = aws_ecs_cluster.main.arn
   ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn
   environment                  = var.environment
-  image_url                    = "${aws_ecr_repository.main.repository_url}:latest"
+  image_url                    = "${aws_ecr_repository.main.repository_url}:${var.image_tag}"
   memory                       = var.index_data_task_memory
   private_subnet_ids           = var.private_subnet_ids
   project_name                 = var.project_name