feat: Added support for persisting Atlantis state using EFS (#247)

Co-authored-by: Anton Babenko <[email protected]>

Author: MarkIannucci

README.md

@@ -248,6 +248,7 @@ allow_github_webhooks        = true
 | <a name="module_container_definition_bitbucket"></a> [container\_definition\_bitbucket](#module\_container\_definition\_bitbucket) | cloudposse/ecs-container-definition/aws | v0.58.1 |
 | <a name="module_container_definition_github_gitlab"></a> [container\_definition\_github\_gitlab](#module\_container\_definition\_github\_gitlab) | cloudposse/ecs-container-definition/aws | v0.58.1 |
 | <a name="module_ecs"></a> [ecs](#module\_ecs) | terraform-aws-modules/ecs/aws | v3.3.0 |
+| <a name="module_efs_sg"></a> [efs\_sg](#module\_efs\_sg) | terraform-aws-modules/security-group/aws//modules/nfs | v4.8.0 |
 | <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | v3.6.0 |
 
 ## Resources
@@ -257,6 +258,9 @@ allow_github_webhooks        = true
 | [aws_cloudwatch_log_group.atlantis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_ecs_service.atlantis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
 | [aws_ecs_task_definition.atlantis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
+| [aws_efs_access_point.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_access_point) | resource |
+| [aws_efs_file_system.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system) | resource |
+| [aws_efs_mount_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_mount_target) | resource |
 | [aws_iam_role.ecs_task_execution](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy.ecs_task_access_secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_iam_role_policy_attachment.ecs_task_execution](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
@@ -337,8 +341,8 @@ allow_github_webhooks        = true
 | <a name="input_ecs_container_insights"></a> [ecs\_container\_insights](#input\_ecs\_container\_insights) | Controls if ECS Cluster has container insights enabled | `bool` | `false` | no |
 | <a name="input_ecs_fargate_spot"></a> [ecs\_fargate\_spot](#input\_ecs\_fargate\_spot) | Whether to run ECS Fargate Spot or not | `bool` | `false` | no |
 | <a name="input_ecs_service_assign_public_ip"></a> [ecs\_service\_assign\_public\_ip](#input\_ecs\_service\_assign\_public\_ip) | Should be true, if ECS service is using public subnets (more info: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_cannot_pull_image.html) | `bool` | `false` | no |
-| <a name="input_ecs_service_deployment_maximum_percent"></a> [ecs\_service\_deployment\_maximum\_percent](#input\_ecs\_service\_deployment\_maximum\_percent) | The upper limit (as a percentage of the service's desiredCount) of the number of running tasks that can be running in a service during a deployment | `number` | `200` | no |
-| <a name="input_ecs_service_deployment_minimum_healthy_percent"></a> [ecs\_service\_deployment\_minimum\_healthy\_percent](#input\_ecs\_service\_deployment\_minimum\_healthy\_percent) | The lower limit (as a percentage of the service's desiredCount) of the number of running tasks that must remain running and healthy in a service during a deployment | `number` | `50` | no |
+| <a name="input_ecs_service_deployment_maximum_percent"></a> [ecs\_service\_deployment\_maximum\_percent](#input\_ecs\_service\_deployment\_maximum\_percent) | The upper limit (as a percentage of the service's desiredCount) of the number of running tasks that can be running in a service during a deployment | `number` | `100` | no |
+| <a name="input_ecs_service_deployment_minimum_healthy_percent"></a> [ecs\_service\_deployment\_minimum\_healthy\_percent](#input\_ecs\_service\_deployment\_minimum\_healthy\_percent) | The lower limit (as a percentage of the service's desiredCount) of the number of running tasks that must remain running and healthy in a service during a deployment | `number` | `0` | no |
 | <a name="input_ecs_service_desired_count"></a> [ecs\_service\_desired\_count](#input\_ecs\_service\_desired\_count) | The number of instances of the task definition to place and keep running | `number` | `1` | no |
 | <a name="input_ecs_service_enable_execute_command"></a> [ecs\_service\_enable\_execute\_command](#input\_ecs\_service\_enable\_execute\_command) | Enable ECS exec for the service. This can be used to allow interactive sessions and commands to be executed in the container | `bool` | `true` | no |
 | <a name="input_ecs_service_force_new_deployment"></a> [ecs\_service\_force\_new\_deployment](#input\_ecs\_service\_force\_new\_deployment) | Enable to force a new task deployment of the service. This can be used to update tasks to use a newer Docker image with same image/tag combination (e.g. myimage:latest) | `bool` | `false` | no |
@@ -379,7 +383,7 @@ allow_github_webhooks        = true
 | <a name="input_trusted_principals"></a> [trusted\_principals](#input\_trusted\_principals) | A list of principals, in addition to ecs-tasks.amazonaws.com, that can assume the task role | `list(string)` | `[]` | no |
 | <a name="input_ulimits"></a> [ulimits](#input\_ulimits) | Container ulimit settings. This is a list of maps, where each map should contain "name", "hardLimit" and "softLimit" | <pre>list(object({<br>    name      = string<br>    hardLimit = number<br>    softLimit = number<br>  }))</pre> | `null` | no |
 | <a name="input_use_ecs_old_arn_format"></a> [use\_ecs\_old\_arn\_format](#input\_use\_ecs\_old\_arn\_format) | A flag to enable/disable tagging the ecs resources that require the new longer arn format | `bool` | `false` | no |
-| <a name="input_user"></a> [user](#input\_user) | The user to run as inside the container. Can be any of these formats: user, user:group, uid, uid:gid, user:gid, uid:group. The default (null) will use the container's configured `USER` directive or root if not set. | `string` | `null` | no |
+| <a name="input_user"></a> [user](#input\_user) | The user to run as inside the container. Must be in the uid:gid or the default (null) will use the container's configured `USER` directive or root if not set. | `string` | `null` | no |
 | <a name="input_volumes_from"></a> [volumes\_from](#input\_volumes\_from) | A list of VolumesFrom maps which contain "sourceContainer" (name of the container that has the volumes to mount) and "readOnly" (whether the container can write to the volume) | <pre>list(object({<br>    sourceContainer = string<br>    readOnly        = bool<br>  }))</pre> | `[]` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | ID of an existing VPC where resources will be created | `string` | `""` | no |
 | <a name="input_webhook_ssm_parameter_name"></a> [webhook\_ssm\_parameter\_name](#input\_webhook\_ssm\_parameter\_name) | Name of SSM parameter to keep webhook secret | `string` | `"/atlantis/webhook/secret"` | no |

examples/github-complete/main.tf

@@ -57,7 +57,6 @@ module "atlantis" {
   start_timeout = 30
   stop_timeout  = 30
 
-  user                     = "atlantis"
   readonly_root_filesystem = false # atlantis currently mutable access to root filesystem
   ulimits = [{
     name      = "nofile"

main.tf

@@ -110,6 +110,17 @@ locals {
     sort(compact(concat(var.allow_github_webhooks ? var.github_webhooks_cidr_blocks : [], var.whitelist_unauthenticated_cidr_blocks))),
     5
   )
+
+  # break up user to uid and gid -- set both to 0 if null
+  uid = var.user == null ? 0 : split(":", var.user)[0]
+  gid = var.user == null ? 0 : split(":", var.user)[1]
+
+  # default mount points for efs if ephemeral storage is not enabled and mount points aren't specified
+  mount_points = var.enable_ephemeral_storage || length(var.mount_points) > 0 ? var.mount_points : [{
+    containerPath = "/home/atlantis"
+    sourceVolume  = "efs-storage"
+    readOnly      = "false"
+  }]
 }
 
 data "aws_partition" "current" {}
@@ -189,8 +200,9 @@ module "vpc" {
   private_subnets = var.private_subnets
   public_subnets  = var.public_subnets
 
-  enable_nat_gateway = true
-  single_nat_gateway = true
+  enable_nat_gateway   = true
+  single_nat_gateway   = true
+  enable_dns_hostnames = !var.enable_ephemeral_storage
 
   manage_default_security_group  = var.manage_default_security_group
   default_security_group_ingress = var.default_security_group_ingress
@@ -355,6 +367,24 @@ module "atlantis_sg" {
   tags = merge(local.tags, var.atlantis_security_group_tags)
 }
 
+module "efs_sg" {
+  source  = "terraform-aws-modules/security-group/aws//modules/nfs"
+  version = "v4.8.0"
+  count   = var.enable_ephemeral_storage ? 0 : 1
+
+  name        = "${var.name}-efs"
+  vpc_id      = local.vpc_id
+  description = "Security group allowing access to the EFS storage"
+
+  ingress_cidr_blocks = [var.cidr]
+  ingress_with_source_security_group_id = [{
+    rule                     = "nfs-tcp",
+    source_security_group_id = module.atlantis_sg.security_group_id
+  }]
+
+  tags = local.tags
+}
+
 ################################################################################
 # ACM (SSL certificate)
 ################################################################################
@@ -388,6 +418,35 @@ resource "aws_route53_record" "atlantis" {
   }
 }
 
+################################################################################
+# EFS
+################################################################################
+
+resource "aws_efs_file_system" "this" {
+  count = var.enable_ephemeral_storage ? 0 : 1
+
+  creation_token = var.name
+}
+
+resource "aws_efs_mount_target" "this" {
+  # we coalescelist in order to specify the resource keys when we create the subnets using the VPC or they're specified for us.  This works around the for_each value depends on attributes which can't be determined until apply error
+  for_each = zipmap(coalescelist(var.private_subnets, var.private_subnet_ids), local.private_subnet_ids)
+
+  file_system_id  = aws_efs_file_system.this[0].id
+  subnet_id       = each.value
+  security_groups = [module.efs_sg[0].security_group_id, module.atlantis_sg.security_group_id]
+}
+
+resource "aws_efs_access_point" "this" {
+  count = var.enable_ephemeral_storage ? 0 : 1
+
+  file_system_id = aws_efs_file_system.this[0].id
+  posix_user {
+    gid = local.gid
+    uid = local.uid
+  }
+}
+
 ################################################################################
 # ECS
 ################################################################################
@@ -521,7 +580,7 @@ module "container_definition_github_gitlab" {
   container_depends_on     = var.container_depends_on
   essential                = var.essential
   readonly_root_filesystem = var.readonly_root_filesystem
-  mount_points             = var.mount_points
+  mount_points             = local.mount_points
   volumes_from             = var.volumes_from
 
   port_mappings = [
@@ -624,11 +683,29 @@ resource "aws_ecs_task_definition" "atlantis" {
 
   dynamic "ephemeral_storage" {
     for_each = var.enable_ephemeral_storage ? [1] : []
+
     content {
       size_in_gib = var.ephemeral_storage_size
     }
   }
 
+  dynamic "volume" {
+    for_each = var.enable_ephemeral_storage ? [] : [1]
+
+    content {
+      name = "efs-storage"
+      efs_volume_configuration {
+        file_system_id          = aws_efs_file_system.this[0].id
+        transit_encryption      = "ENABLED"
+        transit_encryption_port = 2999
+        authorization_config {
+          access_point_id = aws_efs_access_point.this[0].id
+          iam             = "ENABLED"
+        }
+      }
+    }
+  }
+
   tags = local.tags
 }
 
@@ -676,6 +753,7 @@ resource "aws_ecs_service" "atlantis" {
 
   dynamic "capacity_provider_strategy" {
     for_each = var.ecs_fargate_spot ? [true] : []
+
     content {
       capacity_provider = "FARGATE_SPOT"
       weight            = 100

variables.tf

@@ -332,13 +332,13 @@ variable "ecs_service_platform_version" {
 variable "ecs_service_deployment_maximum_percent" {
   description = "The upper limit (as a percentage of the service's desiredCount) of the number of running tasks that can be running in a service during a deployment"
   type        = number
-  default     = 200
+  default     = 100
 }
 
 variable "ecs_service_deployment_minimum_healthy_percent" {
   description = "The lower limit (as a percentage of the service's desiredCount) of the number of running tasks that must remain running and healthy in a service during a deployment"
   type        = number
-  default     = 50
+  default     = 0
 }
 
 variable "ecs_task_cpu" {
@@ -462,9 +462,13 @@ variable "volumes_from" {
 }
 
 variable "user" {
-  description = "The user to run as inside the container. Can be any of these formats: user, user:group, uid, uid:gid, user:gid, uid:group. The default (null) will use the container's configured `USER` directive or root if not set."
+  description = "The user to run as inside the container. Must be in the uid:gid or the default (null) will use the container's configured `USER` directive or root if not set."
   type        = string
   default     = null
+  validation {
+    condition     = can(regex("[0-9]+:[0-9]+", var.user)) || var.user == null
+    error_message = "User variable must be in the uid:gid format or null."
+  }
 }
 
 variable "ulimits" {