feat: parametrize ecs-task cpu/memory limits + aws module 4.0 bump changes (#64)

iru · r2bit · web-flow · commit d9509028a33f · 2022-02-14T11:10:12.000+01:00
* feat(connector): add ability to set CPU and mempry limits for ECS task
* chore: expose ecs cpu/mem variables to examples
* chore: upgrade bucket acl/lifecycle/versioning usage to 4.0
* chore: testing minor changes

Co-authored-by: Janar K &lt;janar.kartau@gmail.com&gt;
diff --git a/README.md b/README.md
@@ -144,6 +144,10 @@ It may take some time, but you should see logs detecting the new image in the EC
 
 ## Troubleshooting
 
+### Q: Getting error "Error: failed creating ECS Task Definition: ClientException: No Fargate configuration exists for given values.
+A: Your ECS task_size values aren't valid for Fargate. Specifically, your mem_limit value is too big for the cpu_limit you specified
+S: Check [supported task cpu and memory values](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html)
+
 ### Q: Getting error "404 Invalid parameter: TopicArn" when trying to reuse an existing cloudtrail-sns
 
 ```text
diff --git a/examples-internal/organizational-k8s-threat-reuse_cloudtrail_s3/README.md b/examples-internal/organizational-k8s-threat-reuse_cloudtrail_s3/README.md
@@ -81,7 +81,7 @@ Notice that:
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 3.74.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.0.0 |
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.4.1 |
 
 ## Modules
diff --git a/examples/organizational/README.md b/examples/organizational/README.md
@@ -74,15 +74,15 @@ Notice that:
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.62.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
 | <a name="requirement_sysdig"></a> [sysdig](#requirement\_sysdig) | >= 0.5.29 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 3.74.1 |
-| <a name="provider_aws.member"></a> [aws.member](#provider\_aws.member) | 3.74.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.0.0 |
+| <a name="provider_aws.member"></a> [aws.member](#provider\_aws.member) | 4.0.0 |
 
 ## Modules
 
@@ -122,6 +122,8 @@ Notice that:
 | <a name="input_deploy_image_scanning_ecr"></a> [deploy\_image\_scanning\_ecr](#input\_deploy\_image\_scanning\_ecr) | true/false whether to deploy the image scanning on ECR pushed images | `bool` | `true` | no |
 | <a name="input_deploy_image_scanning_ecs"></a> [deploy\_image\_scanning\_ecs](#input\_deploy\_image\_scanning\_ecs) | true/false whether to deploy the image scanning on ECS running images | `bool` | `true` | no |
 | <a name="input_ecs_cluster_name"></a> [ecs\_cluster\_name](#input\_ecs\_cluster\_name) | Name of a pre-existing ECS (elastic container service) cluster. If defaulted, a new ECS cluster/VPC/Security Group will be created. For both options, ECS location will/must be within the `sysdig_secure_for_cloud_member_account_id` parameter accountID | `string` | `"create"` | no |
+| <a name="input_ecs_task_cpu"></a> [ecs\_task\_cpu](#input\_ecs\_task\_cpu) | Amount of CPU (in CPU units) to reserve for cloud-connector task | `string` | `"256"` | no |
+| <a name="input_ecs_task_memory"></a> [ecs\_task\_memory](#input\_ecs\_task\_memory) | Amount of memory (in megabytes) to reserve for cloud-connector task | `string` | `"512"` | no |
 | <a name="input_ecs_vpc_id"></a> [ecs\_vpc\_id](#input\_ecs\_vpc\_id) | ID of the VPC where the workload is to be deployed. Defaulted to be created when `ecs_cluster_name is not provided.` | `string` | `"create"` | no |
 | <a name="input_ecs_vpc_region_azs"></a> [ecs\_vpc\_region\_azs](#input\_ecs\_vpc\_region\_azs) | List of Availability Zones for ECS VPC creation. e.g.: ["apne1-az1", "apne1-az2"]. If defaulted, two of the default 'aws\_availability\_zones' datasource will be taken | `list(string)` | `[]` | no |
 | <a name="input_ecs_vpc_subnets_private_ids"></a> [ecs\_vpc\_subnets\_private\_ids](#input\_ecs\_vpc\_subnets\_private\_ids) | List of VPC subnets where workload is to be deployed. Defaulted to be created when `ecs_cluster_name is not provided.` | `list(string)` | `[]` | no |
diff --git a/examples/organizational/main.tf b/examples/organizational/main.tf
@@ -80,6 +80,8 @@ module "cloud_connector" {
   ecs_cluster_name            = local.ecs_cluster_name
   ecs_vpc_id                  = local.ecs_vpc_id
   ecs_vpc_subnets_private_ids = local.ecs_vpc_subnets_private_ids
+  ecs_task_cpu                = var.ecs_task_cpu
+  ecs_task_memory             = var.ecs_task_memory
 
   tags       = var.tags
   depends_on = [local.cloudtrail_sns_arn, module.ssm]
diff --git a/examples/organizational/variables.tf b/examples/organizational/variables.tf
@@ -120,6 +120,20 @@ variable "ecs_vpc_region_azs" {
   default     = []
 }
 
+# Configure CPU and memory in pairs.
+# See: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size
+variable "ecs_task_cpu" {
+  type        = string
+  description = "Amount of CPU (in CPU units) to reserve for cloud-connector task"
+  default     = "256"
+}
+
+variable "ecs_task_memory" {
+  type        = string
+  description = "Amount of memory (in megabytes) to reserve for cloud-connector task"
+  default     = "512"
+}
+
 
 
 #
diff --git a/examples/organizational/versions.tf b/examples/organizational/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_version = ">= 0.15.0"
   required_providers {
     aws = {
-      version = ">= 3.62.0"
+      version = ">= 4.0.0"
     }
     sysdig = {
       source  = "sysdiglabs/sysdig"
diff --git a/examples/single-account-k8s/README.md b/examples/single-account-k8s/README.md
@@ -72,7 +72,7 @@ Notice that:
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 3.74.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.0.0 |
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.4.1 |
 
 ## Modules
diff --git a/examples/single-account/README.md b/examples/single-account/README.md
@@ -50,7 +50,7 @@ Notice that:
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.62.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
 | <a name="requirement_sysdig"></a> [sysdig](#requirement\_sysdig) | >= 0.5.29 |
 
 ## Providers
@@ -86,6 +86,8 @@ No resources.
 | <a name="input_deploy_image_scanning_ecr"></a> [deploy\_image\_scanning\_ecr](#input\_deploy\_image\_scanning\_ecr) | true/false whether to deploy the image scanning on ECR pushed images | `bool` | `true` | no |
 | <a name="input_deploy_image_scanning_ecs"></a> [deploy\_image\_scanning\_ecs](#input\_deploy\_image\_scanning\_ecs) | true/false whether to deploy the image scanning on ECS running images | `bool` | `true` | no |
 | <a name="input_ecs_cluster_name"></a> [ecs\_cluster\_name](#input\_ecs\_cluster\_name) | Name of a pre-existing ECS (elastic container service) cluster. If defaulted, a new ECS cluster/VPC/Security Group will be created | `string` | `"create"` | no |
+| <a name="input_ecs_task_cpu"></a> [ecs\_task\_cpu](#input\_ecs\_task\_cpu) | Amount of CPU (in CPU units) to reserve for cloud-connector task | `string` | `"256"` | no |
+| <a name="input_ecs_task_memory"></a> [ecs\_task\_memory](#input\_ecs\_task\_memory) | Amount of memory (in megabytes) to reserve for cloud-connector task | `string` | `"512"` | no |
 | <a name="input_ecs_vpc_id"></a> [ecs\_vpc\_id](#input\_ecs\_vpc\_id) | ID of the VPC where the workload is to be deployed. Defaulted to be created when 'ecs\_cluster\_name' is not provided. | `string` | `"create"` | no |
 | <a name="input_ecs_vpc_region_azs"></a> [ecs\_vpc\_region\_azs](#input\_ecs\_vpc\_region\_azs) | List of Availability Zones for ECS VPC creation. e.g.: ["apne1-az1", "apne1-az2"]. If defaulted, two of the default 'aws\_availability\_zones' datasource will be taken | `list(string)` | `[]` | no |
 | <a name="input_ecs_vpc_subnets_private_ids"></a> [ecs\_vpc\_subnets\_private\_ids](#input\_ecs\_vpc\_subnets\_private\_ids) | List of VPC subnets where workload is to be deployed. Defaulted to be created when 'ecs\_cluster\_name' is not provided. | `list(string)` | `[]` | no |
diff --git a/examples/single-account/main.tf b/examples/single-account/main.tf
@@ -49,7 +49,8 @@ module "cloud_connector" {
   ecs_cluster_name            = local.ecs_cluster_name
   ecs_vpc_id                  = local.ecs_vpc_id
   ecs_vpc_subnets_private_ids = local.ecs_vpc_subnets_private_ids
-
+  ecs_task_cpu                = var.ecs_task_cpu
+  ecs_task_memory             = var.ecs_task_memory
 
   tags       = var.tags
   depends_on = [local.cloudtrail_sns_arn, module.ssm]
diff --git a/examples/single-account/variables.tf b/examples/single-account/variables.tf
@@ -60,6 +60,19 @@ variable "ecs_vpc_region_azs" {
   default     = []
 }
 
+# Configure CPU and memory in pairs.
+# See: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size
+variable "ecs_task_cpu" {
+  type        = string
+  description = "Amount of CPU (in CPU units) to reserve for cloud-connector task"
+  default     = "256"
+}
+
+variable "ecs_task_memory" {
+  type        = string
+  description = "Amount of memory (in megabytes) to reserve for cloud-connector task"
+  default     = "512"
+}
 
 #
 # scanning configuration
diff --git a/examples/single-account/versions.tf b/examples/single-account/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_version = ">= 0.15.0"
   required_providers {
     aws = {
-      version = ">= 3.62.0"
+      version = ">= 4.0.0"
     }
     sysdig = {
       source  = "sysdiglabs/sysdig"
diff --git a/examples/trigger-events/README.md b/examples/trigger-events/README.md
@@ -47,7 +47,7 @@ Notice that:
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 3.74.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.0.0 |
 
 ## Modules
 
diff --git a/modules/infrastructure/cloudtrail/README.md b/modules/infrastructure/cloudtrail/README.md
@@ -6,13 +6,13 @@
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.50.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.50.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
 
 ## Modules
 
@@ -26,6 +26,8 @@ No modules.
 | [aws_kms_alias.kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource |
 | [aws_kms_key.cloudtrail_kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
 | [aws_s3_bucket.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_bucket_acl.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
+| [aws_s3_bucket_lifecycle_configuration.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource |
 | [aws_s3_bucket_policy.cloudtrail_s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
 | [aws_s3_bucket_public_access_block.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
 | [aws_sns_topic.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource |
diff --git a/modules/infrastructure/cloudtrail/s3.tf b/modules/infrastructure/cloudtrail/s3.tf
@@ -1,18 +1,27 @@
 resource "aws_s3_bucket" "cloudtrail" {
   bucket        = "${var.name}-${data.aws_caller_identity.me.account_id}"
-  acl           = "private"
   force_destroy = true
+  tags          = var.tags
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "cloudtrail" {
+  bucket = aws_s3_bucket.cloudtrail.id
 
-  lifecycle_rule {
-    enabled = true
+  rule {
+    id     = "expire in ${var.s3_bucket_expiration_days} days"
+    status = "Enabled"
     expiration {
       days = var.s3_bucket_expiration_days
     }
   }
-  tags = var.tags
 }
 
 
+resource "aws_s3_bucket_acl" "cloudtrail" {
+  bucket = aws_s3_bucket.cloudtrail.id
+  acl    = "private"
+}
+
 
 # --------------------------
 # iam, acl
diff --git a/modules/infrastructure/cloudtrail/versions.tf b/modules/infrastructure/cloudtrail/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_version = ">= 0.15.0"
   required_providers {
     aws = {
-      version = ">= 3.50.0"
+      version = ">= 4.0.0"
     }
   }
 }
diff --git a/modules/services/cloud-connector/README.md b/modules/services/cloud-connector/README.md
@@ -8,13 +8,13 @@ A task deployed on an **ECS deployment** will detect events in your infrastructu
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.50.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.50.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
 
 ## Modules
 
@@ -39,8 +39,10 @@ A task deployed on an **ECS deployment** will detect events in your infrastructu
 | [aws_iam_role_policy.task_read_parameters](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_iam_role_policy.trigger_scan](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_s3_bucket.s3_config_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
-| [aws_s3_bucket_object.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_object) | resource |
+| [aws_s3_bucket_acl.s3_config_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
 | [aws_s3_bucket_public_access_block.s3_config_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
+| [aws_s3_bucket_versioning.s3_config_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
+| [aws_s3_object.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_security_group.sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_caller_identity.me](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_ecs_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecs_cluster) | data source |
@@ -71,6 +73,8 @@ A task deployed on an **ECS deployment** will detect events in your infrastructu
 | <a name="input_connector_ecs_task_role_name"></a> [connector\_ecs\_task\_role\_name](#input\_connector\_ecs\_task\_role\_name) | Default ecs cloudconnector task role name | `string` | `"ECSTaskRole"` | no |
 | <a name="input_deploy_image_scanning_ecr"></a> [deploy\_image\_scanning\_ecr](#input\_deploy\_image\_scanning\_ecr) | true/false whether to deploy the image scanning on ECR pushed images | `bool` | `true` | no |
 | <a name="input_deploy_image_scanning_ecs"></a> [deploy\_image\_scanning\_ecs](#input\_deploy\_image\_scanning\_ecs) | true/false whether to deploy the image scanning on ECS running images | `bool` | `true` | no |
+| <a name="input_ecs_task_cpu"></a> [ecs\_task\_cpu](#input\_ecs\_task\_cpu) | Amount of CPU (in CPU units) to reserve for cloud-connector task | `string` | `"256"` | no |
+| <a name="input_ecs_task_memory"></a> [ecs\_task\_memory](#input\_ecs\_task\_memory) | Amount of memory (in megabytes) to reserve for cloud-connector task | `string` | `"512"` | no |
 | <a name="input_extra_env_vars"></a> [extra\_env\_vars](#input\_extra\_env\_vars) | Extra environment variables for the Cloud Connector deployment | `map(string)` | `{}` | no |
 | <a name="input_image"></a> [image](#input\_image) | Image of the cloud connector to deploy | `string` | `"quay.io/sysdig/cloud-connector:latest"` | no |
 | <a name="input_is_organizational"></a> [is\_organizational](#input\_is\_organizational) | whether secure-for-cloud should be deployed in an organizational setup | `bool` | `false` | no |
diff --git a/modules/services/cloud-connector/ecs-service.tf b/modules/services/cloud-connector/ecs-service.tf
@@ -24,8 +24,8 @@ resource "aws_ecs_task_definition" "task_definition" {
   network_mode             = "awsvpc"
   execution_role_arn       = aws_iam_role.execution.arn # ARN of the task execution role that the Amazon ECS container agent and the Docker daemon can assume
   task_role_arn            = local.ecs_task_role_arn    # ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS resource-group.
-  cpu                      = "256"
-  memory                   = "512"
+  cpu                      = var.ecs_task_cpu
+  memory                   = var.ecs_task_memory
 
   container_definitions = jsonencode([
     {
diff --git a/modules/services/cloud-connector/s3-config.tf b/modules/services/cloud-connector/s3-config.tf
@@ -2,9 +2,10 @@ locals {
   s3_bucket_config_id = aws_s3_bucket.s3_config_bucket.id
 }
 
-resource "aws_s3_bucket_object" "config" {
-  bucket  = local.s3_bucket_config_id
-  key     = "cloud-connector.yaml"
+resource "aws_s3_object" "config" {
+  bucket = local.s3_bucket_config_id
+  key    = "cloud-connector.yaml"
+
   content = local.default_config
   tags    = var.tags
 }
diff --git a/modules/services/cloud-connector/s3.tf b/modules/services/cloud-connector/s3.tf
@@ -2,12 +2,22 @@ data "aws_caller_identity" "me" {}
 
 resource "aws_s3_bucket" "s3_config_bucket" {
   bucket        = "${var.name}-${data.aws_caller_identity.me.account_id}-config"
-  acl           = "private"
   force_destroy = true
-  versioning {
-    enabled = true
+  tags          = var.tags
+}
+
+
+resource "aws_s3_bucket_acl" "s3_config_bucket" {
+  bucket = aws_s3_bucket.s3_config_bucket.id
+  acl    = "private"
+}
+
+
+resource "aws_s3_bucket_versioning" "s3_config_bucket" {
+  bucket = aws_s3_bucket.s3_config_bucket.id
+  versioning_configuration {
+    status = "Enabled"
   }
-  tags = var.tags
 }
 
 
diff --git a/modules/services/cloud-connector/variables.tf b/modules/services/cloud-connector/variables.tf
@@ -47,7 +47,6 @@ variable "sns_topic_arn" {
 # optionals - with default
 #---------------------------------
 
-
 #
 # module composition
 #
@@ -84,6 +83,21 @@ variable "organizational_config" {
 #
 # module config
 #
+
+# Configure CPU and memory in pairs.
+# See: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size
+variable "ecs_task_cpu" {
+  type        = string
+  description = "Amount of CPU (in CPU units) to reserve for cloud-connector task"
+  default     = "256"
+}
+
+variable "ecs_task_memory" {
+  type        = string
+  description = "Amount of memory (in megabytes) to reserve for cloud-connector task"
+  default     = "512"
+}
+
 variable "connector_ecs_task_role_name" {
   type        = string
   default     = "ECSTaskRole"
diff --git a/modules/services/cloud-connector/versions.tf b/modules/services/cloud-connector/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_version = ">= 0.15.0"
   required_providers {
     aws = {
-      version = ">= 3.50.0"
+      version = ">= 4.0.0"
     }
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@ terraform {`
`2`	`2`	`required_version = ">= 0.15.0"`
`3`	`3`	`required_providers {`
`4`	`4`	`aws = {`
`5`		`- version = ">= 3.62.0"`
	`5`	`+ version = ">= 4.0.0"`
`6`	`6`	`}`
`7`	`7`	`sysdig = {`
`8`	`8`	`source = "sysdiglabs/sysdig"`