Manage the lambda log groups via terraform (#126)

Author: npalm

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- feat: Manage log groups via module. When upgrading you have to import the log groups by AWS into your state. See below the example commands for the default example.
+```bash
+terraform import module.runners.module.runner_binaries.aws_cloudwatch_log_group.syncer "/aws/lambda/default-syncer"
+terraform import module.runners.module.runners.aws_cloudwatch_log_group.scale_up "/aws/lambda/default-scale-up"
+terraform import module.runners.module.runners.aws_cloudwatch_log_group.scale_down "/aws/lambda/default-scale-down"
+terraform import module.runners.module.webhook.aws_cloudwatch_log_group.webhook "/aws/lambda/default-webhook"
+```
+
 ## [0.4.0] - 2020-08-10
 
 ### Added

README.md

@@ -38,8 +38,9 @@ module "webhook" {
   sqs_build_queue           = aws_sqs_queue.queued_builds
   github_app_webhook_secret = var.github_app.webhook_secret
 
-  lambda_zip     = var.webhook_lambda_zip
-  lambda_timeout = var.webhook_lambda_timeout
+  lambda_zip                = var.webhook_lambda_zip
+  lambda_timeout            = var.webhook_lambda_timeout
+  logging_retention_in_days = var.logging_retention_in_days
 
   role_path                 = var.role_path
   role_permissions_boundary = var.role_permissions_boundary
@@ -79,6 +80,7 @@ module "runners" {
   lambda_zip                = var.runners_lambda_zip
   lambda_timeout_scale_up   = var.runners_scale_up_lambda_timeout
   lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout
+  logging_retention_in_days = var.logging_retention_in_days
 
   instance_profile_path     = var.instance_profile_path
   role_path                 = var.role_path
@@ -99,8 +101,9 @@ module "runner_binaries" {
 
   runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64"
 
-  lambda_zip     = var.runner_binaries_syncer_lambda_zip
-  lambda_timeout = var.runner_binaries_syncer_lambda_timeout
+  lambda_zip                = var.runner_binaries_syncer_lambda_zip
+  lambda_timeout            = var.runner_binaries_syncer_lambda_timeout
+  logging_retention_in_days = var.logging_retention_in_days
 
   role_path                 = var.role_path
   role_permissions_boundary = var.role_permissions_boundary

main.tf

@@ -54,6 +54,7 @@ No requirements.
 | lambda\_schedule\_expression | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no |
 | lambda\_timeout | Time out of the lambda in seconds. | `number` | `300` | no |
 | lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
+| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
 | role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
 | role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
 | runner\_architecture | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |

modules/runner-binaries-syncer/README.md

@@ -3,12 +3,8 @@
   "Statement": [
     {
       "Effect": "Allow",
-      "Action": [
-        "logs:CreateLogGroup",
-        "logs:CreateLogStream",
-        "logs:PutLogEvents"
-      ],
-      "Resource": "arn:aws:logs:*:*:*"
+      "Action": ["logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "${log_group_arn}"
     }
   ]
 }

modules/runner-binaries-syncer/policies/lambda-cloudwatch.json

@@ -23,6 +23,12 @@ resource "aws_lambda_function" "syncer" {
   tags = var.tags
 }
 
+resource "aws_cloudwatch_log_group" "syncer" {
+  name              = "/aws/lambda/${aws_lambda_function.syncer.function_name}"
+  retention_in_days = var.logging_retention_in_days
+  tags              = var.tags
+}
+
 resource "aws_iam_role" "syncer_lambda" {
   name                 = "${var.environment}-action-syncer-lambda-role"
   assume_role_policy   = data.aws_iam_policy_document.lambda_assume_role_policy.json
@@ -47,7 +53,9 @@ resource "aws_iam_role_policy" "lambda_logging" {
   name = "${var.environment}-lambda-logging-policy-syncer"
   role = aws_iam_role.syncer_lambda.id
 
-  policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {})
+  policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {
+    log_group_arn = aws_cloudwatch_log_group.syncer.arn
+  })
 }
 
 resource "aws_iam_role_policy" "syncer" {

modules/runner-binaries-syncer/runner-binaries-syncer.tf

@@ -54,3 +54,9 @@ variable "runner_architecture" {
   type        = string
   default     = "x64"
 }
+
+variable "logging_retention_in_days" {
+  description = "Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653."
+  type        = number
+  default     = 7
+}

modules/runner-binaries-syncer/variables.tf

@@ -76,6 +76,7 @@ No requirements.
 | lambda\_timeout\_scale\_down | Time out for the scale down lambda in seconds. | `number` | `60` | no |
 | lambda\_timeout\_scale\_up | Time out for the scale up lambda in seconds. | `number` | `60` | no |
 | lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
+| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
 | market\_options | Market options for the action runner instances. | `string` | `"spot"` | no |
 | minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
 | overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` override the `Name` tag spot instances created by the runner agent. | `map(string)` | <pre>{<br>  "name_runner": "",<br>  "name_sg": ""<br>}</pre> | no |

modules/runners/README.md

@@ -3,12 +3,8 @@
   "Statement": [
     {
       "Effect": "Allow",
-      "Action": [
-        "logs:CreateLogGroup",
-        "logs:CreateLogStream",
-        "logs:PutLogEvents"
-      ],
-      "Resource": "arn:aws:logs:*:*:*"
+      "Action": ["logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "${log_group_arn}"
     }
   ]
 }

modules/runners/policies/lambda-cloudwatch.json

@@ -37,6 +37,12 @@ resource "aws_lambda_function" "scale_down" {
   }
 }
 
+resource "aws_cloudwatch_log_group" "scale_down" {
+  name              = "/aws/lambda/${aws_lambda_function.scale_down.function_name}"
+  retention_in_days = var.logging_retention_in_days
+  tags              = var.tags
+}
+
 resource "aws_cloudwatch_event_rule" "scale_down" {
   name                = "${var.environment}-scale-down-rule"
   schedule_expression = var.scale_down_schedule_expression
@@ -71,7 +77,9 @@ resource "aws_iam_role_policy" "scale_down" {
 }
 
 resource "aws_iam_role_policy" "scale_down_logging" {
-  name   = "${var.environment}-lambda-logging"
-  role   = aws_iam_role.scale_down.name
-  policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {})
+  name = "${var.environment}-lambda-logging"
+  role = aws_iam_role.scale_down.name
+  policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {
+    log_group_arn = aws_cloudwatch_log_group.scale_down.arn
+  })
 }