WIP: attach lambda to SQS queue

Author: gertjanmaas

main.tf

@@ -23,6 +23,16 @@ module "runners" {
   tags        = var.tags
 
   s3_location_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution
+  sqs                             = module.agent.sqs
+}
+
+module "agent" {
+  source = "./modules/agent"
+
+  aws_region                = var.aws_region
+  environment               = var.environment
+  tags                      = var.tags
+  github_app_webhook_secret = "blaat"
 }
 
 

modules/agent/outputs.tf

@@ -1,3 +1,7 @@
 output "gateway" {
   value = aws_apigatewayv2_api.webhook
 }
+
+output "sqs" {
+  value = aws_sqs_queue.webhook_events
+}

modules/agent/webhook.tf

@@ -38,11 +38,12 @@ resource "aws_apigatewayv2_integration" "webhook" {
 
 
 resource "aws_lambda_function" "webhook" {
-  filename      = "webhook.zip"
-  function_name = "${var.environment}-webhook"
-  role          = aws_iam_role.webhook_lambda.arn
-  handler       = "lambda.githubWebhook"
-  runtime       = "nodejs12.x"
+  filename         = "${path.module}/lambdas/webhook/webhook.zip"
+  source_code_hash = filebase64sha256("${path.module}/lambdas/webhook/webhook.zip")
+  function_name    = "${var.environment}-webhook"
+  role             = aws_iam_role.webhook_lambda.arn
+  handler          = "index.githubWebhook"
+  runtime          = "nodejs12.x"
 
   environment {
     variables = {
@@ -85,6 +86,3 @@ resource "aws_iam_policy_attachment" "webhook" {
   roles      = [aws_iam_role.webhook_lambda.name]
   policy_arn = aws_iam_policy.webhook.arn
 }
-
-
-

modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts

@@ -1,3 +1,5 @@
+import { IncomingHttpHeaders } from 'http';
+
 export const handle = async (headers: IncomingHttpHeaders, payload: any): Promise<number> => {
   return 200;
 };

modules/runners/lambdas/scale-runners/test/resources/sqs_receive_event.json

@@ -0,0 +1,23 @@
+{
+  "Records": [
+    {
+      "messageId": "f7f4e155-2079-4255-b7a0-1b7b4be45ff9",
+      "receiptHandle": "AQEBpE+kwApifOOwbeTp0xFbeOOjnPTHvMCPFIbft3ah3C50GAUD2RKz3ZzVKFxFRdD50uHrKt7rKpDHCuavO5TBj9Gql7YH6G4iR9Vqz9XFFAQQGlcHf+EfVsDAewPr0FLiW40ZC+mNNGwYh9Bqbo5MAmpNWxYWImI4VIEGknW0oFLMSSVd6js7eSkRaJoL5belvjl06b48b/PUvyk0Su367xTTRsf6esih3ALb9RBI0ylV78kmDEQLcNi/7X1pA3UChQcvEn5+bp5JKlhalQRFDyRqMmZr7KeUDI/vG2gbMHOWuLkwzTl5jsKGc/pPVi86",
+      "body": "{\"id\":128620228,\"repositoryName\":\"Hello-World\",\"repositoryOwner\":\"Codertocat\",\"eventType\":\"check_run\",\"installationId\":12345}",
+      "attributes": {
+        "ApproximateReceiveCount": "1",
+        "SentTimestamp": "1588152306469",
+        "SequenceNumber": "18853311064165616128",
+        "MessageGroupId": "128620228",
+        "SenderId": "AROAVQMGTCYMGIEWL5JV5:default-action-runners-webhook",
+        "MessageDeduplicationId": "bdc9a81e515df0131ddc015b1182b57ffcd79b0321bfe32bb40572b23ee68c50",
+        "ApproximateFirstReceiveTimestamp": "1588152306469"
+      },
+      "messageAttributes": {},
+      "md5OfBody": "f30235cb7733c3ac59a14d99c59a6dbf",
+      "eventSource": "aws:sqs",
+      "eventSourceARN": "arn:aws:sqs:eu-west-1:378776262168:default-action-runners-webhook-events.fifo",
+      "awsRegion": "eu-west-1"
+    }
+  ]
+}

modules/runners/policies/lambda-cloudwatch.json

@@ -0,0 +1,14 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*"
+    }
+  ]
+}

modules/runners/policies/lambda-scale-runners.json

@@ -0,0 +1,14 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:ReceiveMessage",
+        "sqs:GetQueueAttributes",
+        "sqs:DeleteMessage"
+      ],
+      "Resource": "${sqs_arn}"
+    }
+  ]
+}

modules/runners/scale-runners-lambda.tf

@@ -0,0 +1,70 @@
+resource "aws_lambda_function" "scale_runners_lambda" {
+  filename         = "${path.module}/lambdas/scale-runners/scale-runners.zip"
+  source_code_hash = filebase64sha256("${path.module}/lambdas/scale-runners/scale-runners.zip")
+  function_name    = "${var.environment}-scale-runners"
+  role             = aws_iam_role.scale_runners_lambda.arn
+  handler          = "index.handler"
+  runtime          = "nodejs12.x"
+
+  # environment {
+  #   variables = {
+  #   }
+  # }
+}
+
+resource "aws_lambda_event_source_mapping" "scale_runners_lambda" {
+  event_source_arn = var.sqs.arn
+  function_name    = aws_lambda_function.scale_runners_lambda.arn
+}
+
+resource "aws_lambda_permission" "scale_runners_lambda" {
+  statement_id  = "AllowExecutionFromSQS"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.scale_runners_lambda.function_name
+  principal     = "sqs.amazonaws.com"
+  source_arn    = var.sqs.arn
+}
+
+resource "aws_iam_role" "scale_runners_lambda" {
+  name               = "${var.environment}-action-scale-runners-lambda-role"
+  assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json
+}
+
+data "aws_iam_policy_document" "lambda_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_policy" "lambda_logging" {
+  name        = "${var.environment}-lamda-runners-logging-policy"
+  description = "Lambda logging policy"
+
+  policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {})
+}
+
+resource "aws_iam_policy_attachment" "scale_runners_lambda_logging" {
+  name       = "${var.environment}-logging"
+  roles      = [aws_iam_role.scale_runners_lambda.name]
+  policy_arn = aws_iam_policy.lambda_logging.arn
+}
+
+resource "aws_iam_policy" "scale_runners_lambda" {
+  name        = "${var.environment}-lamda-scale-runners-sqs-receive-policy"
+  description = "Lambda webhook policy"
+
+  policy = templatefile("${path.module}/policies/lambda-scale-runners.json", {
+    sqs_arn = var.sqs.arn
+  })
+}
+
+resource "aws_iam_policy_attachment" "scale_runners_lambda" {
+  name       = "${var.environment}-scale-runners"
+  roles      = [aws_iam_role.scale_runners_lambda.name]
+  policy_arn = aws_iam_policy.scale_runners_lambda.arn
+}

modules/runners/variables.tf

@@ -78,3 +78,4 @@ variable "userdata_post_install" {
   type        = string
   default     = ""
 }
+variable "sqs" {}

outputs.tf

@@ -4,5 +4,6 @@ output "runners" {
     launch_template_id         = module.runners.launch_template.id
     launch_template_version    = module.runners.launch_template.latest_version
     action_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution
+    gateway                    = module.agent.gateway
   }
 }