Feature/log to cloudwatch (#375)

* feat: stream logging to cloudwatch of the agent, user_data en system messages

* Refactor and make cloudwatch agent optional

* Apply suggestions from code review

Co-authored-by: Gertjan Maas <[email protected]>

* Update docs

Co-authored-by: Gertjan Maas <[email protected]>

Author: npalm

README.md

@@ -92,6 +92,9 @@ module "runners" {
   lambda_timeout_scale_up          = var.runners_scale_up_lambda_timeout
   lambda_timeout_scale_down        = var.runners_scale_down_lambda_timeout
   logging_retention_in_days        = var.logging_retention_in_days
+  enable_cloudwatch_agent          = var.enable_cloudwatch_agent
+  cloudwatch_config                = var.cloudwatch_config
+  runner_log_files                 = var.runner_log_files
 
   instance_profile_path     = var.instance_profile_path
   role_path                 = var.role_path

main.tf

@@ -66,7 +66,9 @@ No requirements.
 | ami\_owners | The list of owners used to select the AMI of action runner instances. | `list(string)` | <pre>[<br>  "amazon"<br>]</pre> | no |
 | aws\_region | AWS region. | `string` | n/a | yes |
 | block\_device\_mappings | The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops` | `map(string)` | `{}` | no |
+| cloudwatch\_config | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | `string` | `null` | no |
 | create\_service\_linked\_role\_spot | (optional) create the serviced linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no |
+| enable\_cloudwatch\_agent | Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`. | `bool` | `true` | no |
 | enable\_organization\_runners | n/a | `bool` | n/a | yes |
 | enable\_ssm\_on\_runners | Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | n/a | yes |
 | encryption | KMS key to encrypted lambda environment secrets. Either provide a key and `encrypt` set to `true`. Or set the key to `null` and encrypt to `false`. | <pre>object({<br>    kms_key_id = string<br>    encrypt    = bool<br>  })</pre> | n/a | yes |
@@ -88,6 +90,8 @@ No requirements.
 | runner\_architecture | The platform architecture of the runner instance\_type. | `string` | `"x64"` | no |
 | runner\_as\_root | Run the action runner under the root user. | `bool` | `false` | no |
 | runner\_extra\_labels | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
+| runner\_iam\_role\_managed\_policy\_arns | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
+| runner\_log\_files | (optional) List of logfiles to send to cloudwatch. | <pre>list(object({<br>    file_path       = string<br>    log_stream_name = string<br>  }))</pre> | <pre>[<br>  {<br>    "file_path": "/var/log/messages",<br>    "log_stream_name": "{instance_id}/messages"<br>  },<br>  {<br>    "file_path": "/var/log/user-data.log",<br>    "log_stream_name": "{instance_id}/user_data"<br>  },<br>  {<br>    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br>    "log_stream_name": "{instance_id}/runner"<br>  }<br>]</pre> | no |
 | runners\_lambda\_s3\_key | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
 | runners\_lambda\_s3\_object\_version | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
 | runners\_maximum\_count | The maximum number of runners that will be created. | `number` | `3` | no |

modules/runners/README.md

@@ -0,0 +1,20 @@
+locals {
+  logfiles = var.enable_cloudwatch_agent ? [for l in var.runner_log_files : merge(l, { "log_group_name" : aws_cloudwatch_log_group.runners[0].name })] : []
+}
+
+resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" {
+  count = var.enable_cloudwatch_agent ? 1 : 0
+  name  = "${var.environment}-cloudwatch_agent_config_runner"
+  type  = "String"
+  value = var.cloudwatch_config != null ? var.cloudwatch_config : templatefile("${path.module}/templates/cloudwatch_config.json", {
+    logfiles = jsonencode(local.logfiles)
+  })
+  tags = local.tags
+}
+
+resource "aws_cloudwatch_log_group" "runners" {
+  count             = var.enable_cloudwatch_agent ? 1 : 0
+  name              = "${var.environment}/runners"
+  retention_in_days = var.logging_retention_in_days
+  tags              = local.tags
+}

modules/runners/logging.tf

@@ -81,6 +81,8 @@ resource "aws_launch_template" "runner" {
     s3_location_runner_distribution = var.s3_location_runner_binaries
     service_user                    = var.runner_as_root ? "root" : "ec2-user"
     runner_architecture             = var.runner_architecture
+    enable_cloudwatch_agent         = var.enable_cloudwatch_agent
+    ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : ""
   }))
 
   tags = local.tags

modules/runners/main.tf

@@ -0,0 +1,12 @@
+{
+    "agent": {
+        "metrics_collection_interval": 5
+    },
+    "logs": {
+        "logs_collected": {
+            "files": {
+                "collect_list": ${logfiles}
+            }
+        }
+    }
+}

modules/runners/templates/cloudwatch_config.json

@@ -5,6 +5,11 @@ ${pre_install}
 
 yum update -y
 
+%{ if enable_cloudwatch_agent ~}
+yum install amazon-cloudwatch-agent -y
+amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${ssm_key_cloudwatch_agent_config}
+%{ endif ~}
+
 # Install docker
 amazon-linux-extras install docker
 service docker start
@@ -77,8 +82,8 @@ REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq
 
 echo wait for configuration
 while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do
-    echo Waiting for configuration ...
-    sleep 1
+  echo Waiting for configuration ...
+  sleep 1
 done
 CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value")
 aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION

modules/runners/templates/user-data.sh

@@ -244,3 +244,37 @@ variable "runner_iam_role_managed_policy_arns" {
   type        = list(string)
   default     = []
 }
+
+variable "enable_cloudwatch_agent" {
+  description = "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
+  type        = bool
+  default     = true
+}
+
+variable "cloudwatch_config" {
+  description = "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
+  type        = string
+  default     = null
+}
+
+variable "runner_log_files" {
+  description = "(optional) List of logfiles to send to cloudwatch."
+  type = list(object({
+    file_path       = string
+    log_stream_name = string
+  }))
+  default = [
+    {
+      "file_path" : "/var/log/messages",
+      "log_stream_name" : "{instance_id}/messages"
+    },
+    {
+      "file_path" : "/var/log/user-data.log",
+      "log_stream_name" : "{instance_id}/user_data"
+    },
+    {
+      "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log",
+      "log_stream_name" : "{instance_id}/runner"
+    }
+  ]
+}

modules/runners/variables.tf

@@ -262,3 +262,37 @@ variable "runner_iam_role_managed_policy_arns" {
   type        = list(string)
   default     = []
 }
+
+variable "enable_cloudwatch_agent" {
+  description = "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
+  type        = bool
+  default     = true
+}
+
+variable "cloudwatch_config" {
+  description = "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
+  type        = string
+  default     = null
+}
+
+variable "runner_log_files" {
+  description = "(optional) List of logfiles to send to cloudwatch."
+  type = list(object({
+    file_path       = string
+    log_stream_name = string
+  }))
+  default = [
+    {
+      "file_path" : "/var/log/messages",
+      "log_stream_name" : "{instance_id}/messages"
+    },
+    {
+      "file_path" : "/var/log/user-data.log",
+      "log_stream_name" : "{instance_id}/user_data"
+    },
+    {
+      "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log",
+      "log_stream_name" : "{instance_id}/runner"
+    }
+  ]
+}