[eks/actions-runner-controller] Auth via GitHub App, prefer webhook auto-scaling (cloudposse/terraform-aws-components#519)

Author: Nuru

src/README.md

@@ -1,23 +1,28 @@
 locals {
-  enabled               = module.this.enabled
-  identity_account_name = module.account_map.outputs.identity_account_account_name
-  identity_account_id   = module.account_map.outputs.full_account_map[local.identity_account_name]
-  stack_name            = local.enabled ? format("${module.this.tenant != null ? "%[1]s-" : ""}%[2]s-%[3]s", module.this.tenant, module.this.environment, module.this.stage) : ""
-  webhook_enabled       = local.enabled ? try(var.webhook.enabled, false) : false
-  webhook_host          = local.webhook_enabled ? format(var.webhook.hostname_template, var.tenant, var.stage, var.environment) : "example.com"
-
-  default_secrets = local.enabled ? [
+  enabled = module.this.enabled
+
+  webhook_enabled = local.enabled ? try(var.webhook.enabled, false) : false
+  webhook_host    = local.webhook_enabled ? format(var.webhook.hostname_template, var.tenant, var.stage, var.environment) : "example.com"
+
+  github_app_enabled = length(var.github_app_id) > 0 && length(var.github_app_installation_id) > 0
+  create_secret      = local.enabled && length(var.existing_kubernetes_secret_name) == 0
+
+  busy_metrics_filtered = { for runner, runner_config in var.runners : runner => try(runner_config.busy_metrics, null) == null ? null : {
+    for k, v in runner_config.busy_metrics : k => v if v != null
+  } }
+
+  default_secrets = local.create_secret ? [
     {
-      name  = "authSecret.github_token"
-      value = join("", data.aws_ssm_parameter.github_token[0].*.value)
+      name  = local.github_app_enabled ? "authSecret.github_app_private_key" : "authSecret.github_token"
+      value = one(data.aws_ssm_parameter.github_token[*].value)
       type  = "string"
     }
   ] : []
 
-  webhook_secrets = local.webhook_enabled ? [
+  webhook_secrets = local.create_secret && local.webhook_enabled ? [
     {
       name  = "githubWebhookServer.secret.github_webhook_secret_token"
-      value = join("", data.aws_ssm_parameter.github_webhook_secret_token[0].*.value)
+      value = one(data.aws_ssm_parameter.github_webhook_secret_token[*].value)
       type  = "string"
     }
   ] : []
@@ -80,39 +85,36 @@ locals {
   iam_policy_statements = concat(local.default_iam_policy_statements, local.s3_iam_policy_statements)
 }
 
-data "aws_partition" "current" {
-  count = local.enabled ? 1 : 0
-}
-
 data "aws_ssm_parameter" "github_token" {
-  count = local.enabled ? 1 : 0
+  count = local.create_secret ? 1 : 0
 
-  name            = var.ssm_github_token_path
+  name            = var.ssm_github_secret_path
   with_decryption = true
 }
 
 data "aws_ssm_parameter" "github_webhook_secret_token" {
-  count = local.webhook_enabled ? 1 : 0
+  count = local.create_secret && local.webhook_enabled ? 1 : 0
 
   name            = var.ssm_github_webhook_secret_token_path
   with_decryption = true
 }
 
 module "actions_runner_controller" {
   source  = "cloudposse/helm-release/aws"
-  version = "0.6.0"
+  version = "0.7.0"
 
-  name                 = "" # avoids hitting length restrictions on IAM Role names
-  chart                = var.chart
-  repository           = var.chart_repository
-  description          = var.chart_description
-  chart_version        = var.chart_version
-  kubernetes_namespace = var.kubernetes_namespace
-  create_namespace     = var.create_namespace
-  wait                 = var.wait
-  atomic               = var.atomic
-  cleanup_on_fail      = var.cleanup_on_fail
-  timeout              = var.timeout
+  name            = "" # avoids hitting length restrictions on IAM Role names
+  chart           = var.chart
+  repository      = var.chart_repository
+  description     = var.chart_description
+  chart_version   = var.chart_version
+  wait            = var.wait
+  atomic          = var.atomic
+  cleanup_on_fail = var.cleanup_on_fail
+  timeout         = var.timeout
+
+  kubernetes_namespace             = var.kubernetes_namespace
+  create_namespace_with_kubernetes = var.create_namespace
 
   eks_cluster_oidc_issuer_url = module.eks.outputs.eks_cluster_identity_oidc_issuer
 
@@ -155,7 +157,23 @@ module "actions_runner_controller" {
       },
       authSecret = {
         enabled = true
-        create  = true
+        create  = local.create_secret
+      }
+    }),
+    local.github_app_enabled ? yamlencode({
+      authSecret = {
+        github_app_id              = var.github_app_id
+        github_app_installation_id = var.github_app_installation_id
+      }
+    }) : "",
+    local.create_secret ? "" : yamlencode({
+      authSecret = {
+        name = var.existing_kubernetes_secret_name
+      },
+      githubWebhookServer = {
+        secret = {
+          name = var.existing_kubernetes_secret_name
+        }
       }
     }),
     # additional values
@@ -171,18 +189,18 @@ module "actions_runner" {
   for_each = local.enabled ? var.runners : {}
 
   source  = "cloudposse/helm-release/aws"
-  version = "0.6.0"
+  version = "0.7.0"
 
   name  = each.key
   chart = "${path.module}/charts/actions-runner"
 
   kubernetes_namespace = var.kubernetes_namespace
-  create_namespace     = var.create_namespace
+  create_namespace     = false # will be created by controller above
   atomic               = var.atomic
 
   eks_cluster_oidc_issuer_url = module.eks.outputs.eks_cluster_identity_oidc_issuer
 
-  values = [
+  values = compact([
     yamlencode({
       release_name                   = each.key
       service_account_name           = module.actions_runner_controller.service_account_name
@@ -197,16 +215,11 @@ module "actions_runner" {
       scale_down_delay_seconds       = each.value.scale_down_delay_seconds
       min_replicas                   = each.value.min_replicas
       max_replicas                   = each.value.max_replicas
-      scale_up_threshold             = try(each.value.busy_metrics.scale_up_threshold, null)
-      scale_down_threshold           = try(each.value.busy_metrics.scale_down_threshold, null)
-      scale_up_adjustment            = try(each.value.busy_metrics.scale_up_adjustment, null)
-      scale_down_adjustment          = try(each.value.busy_metrics.scale_down_adjustment, null)
-      scale_up_factor                = try(each.value.busy_metrics.scale_up_factor, null)
-      scale_down_factor              = try(each.value.busy_metrics.scale_down_factor, null)
       webhook_driven_scaling_enabled = each.value.webhook_driven_scaling_enabled
       pull_driven_scaling_enabled    = each.value.pull_driven_scaling_enabled
-    })
-  ]
+    }),
+    local.busy_metrics_filtered[each.key] == null ? "" : yamlencode(local.busy_metrics_filtered[each.key]),
+  ])
 
   depends_on = [module.actions_runner_controller]
 }

src/main.tf

@@ -2,6 +2,12 @@
 #
 # This file is a drop-in to provide a helm provider.
 #
+# It depends on 2 standard Cloud Posse data source modules to be already
+# defined in the same component:
+#
+#   1. module.iam_roles to provide the AWS profile or Role ARN to use to access the cluster
+#   2. module.eks to provide the EKS cluster information
+#
 # All the following variables are just about configuring the Kubernetes provider
 # to be able to modify EKS cluster. The reason there are so many options is
 # because at various times, each one of them has had problems, so we give you a choice.
@@ -100,9 +106,11 @@ locals {
     "--role-arn", local.kube_exec_auth_role_arn
   ] : []
 
-  certificate_authority_data = module.eks.outputs.eks_cluster_certificate_authority_data
-  eks_cluster_id             = module.eks.outputs.eks_cluster_id
-  eks_cluster_endpoint       = module.eks.outputs.eks_cluster_endpoint
+  # Provide dummy configuration for the case where the EKS cluster is not available.
+  certificate_authority_data = try(module.eks.outputs.eks_cluster_certificate_authority_data, "")
+  # Use coalesce+try to handle both the case where the output is missing and the case where it is empty.
+  eks_cluster_id       = coalesce(try(module.eks.outputs.eks_cluster_id, ""), "missing")
+  eks_cluster_endpoint = try(module.eks.outputs.eks_cluster_endpoint, "")
 }
 
 data "aws_eks_cluster_auth" "eks" {
@@ -114,14 +122,14 @@ provider "helm" {
   kubernetes {
     host                   = local.eks_cluster_endpoint
     cluster_ca_certificate = base64decode(local.certificate_authority_data)
-    token                  = local.kube_data_auth_enabled ? data.aws_eks_cluster_auth.eks[0].token : null
+    token                  = local.kube_data_auth_enabled ? one(data.aws_eks_cluster_auth.eks[*].token) : null
     # The Kubernetes provider will use information from KUBECONFIG if it exists, but if the default cluster
     # in KUBECONFIG is some other cluster, this will cause problems, so we override it always.
     config_path    = local.kubeconfig_file_enabled ? var.kubeconfig_file : ""
     config_context = var.kubeconfig_context
 
     dynamic "exec" {
-      for_each = local.kube_exec_auth_enabled ? ["exec"] : []
+      for_each = local.kube_exec_auth_enabled && length(local.certificate_authority_data) > 0 ? ["exec"] : []
       content {
         api_version = local.kubeconfig_exec_auth_api_version
         command     = "aws"
@@ -132,21 +140,21 @@ provider "helm" {
     }
   }
   experiments {
-    manifest = var.helm_manifest_experiment_enabled
+    manifest = var.helm_manifest_experiment_enabled && module.this.enabled
   }
 }
 
 provider "kubernetes" {
   host                   = local.eks_cluster_endpoint
   cluster_ca_certificate = base64decode(local.certificate_authority_data)
-  token                  = local.kube_data_auth_enabled ? data.aws_eks_cluster_auth.eks[0].token : null
+  token                  = local.kube_data_auth_enabled ? one(data.aws_eks_cluster_auth.eks[*].token) : null
   # The Kubernetes provider will use information from KUBECONFIG if it exists, but if the default cluster
   # in KUBECONFIG is some other cluster, this will cause problems, so we override it always.
   config_path    = local.kubeconfig_file_enabled ? var.kubeconfig_file : ""
   config_context = var.kubeconfig_context
 
   dynamic "exec" {
-    for_each = local.kube_exec_auth_enabled ? ["exec"] : []
+    for_each = local.kube_exec_auth_enabled && length(local.certificate_authority_data) > 0 ? ["exec"] : []
     content {
       api_version = local.kubeconfig_exec_auth_api_version
       command     = "aws"

src/provider-helm.tf

@@ -1,20 +1,8 @@
 module "eks" {
   source  = "cloudposse/stack-config/yaml//modules/remote-state"
-  version = "0.22.4"
+  version = "1.3.1"
 
   component = var.eks_component_name
 
   context = module.this.context
 }
-
-module "account_map" {
-  source  = "cloudposse/stack-config/yaml//modules/remote-state"
-  version = "0.22.4"
-
-  component   = "account-map"
-  environment = var.account_map_environment_name
-  stage       = var.account_map_stage_name
-  tenant      = var.account_map_tenant_name
-
-  context = module.this.context
-}

src/remote-state.tf

@@ -88,6 +88,7 @@ variable "rbac_enabled" {
 
 # Runner-specific settings
 
+/*
 variable "account_map_environment_name" {
   type        = string
   description = "The name of the environment where `account_map` is provisioned"
@@ -110,6 +111,20 @@ variable "account_map_tenant_name" {
   default     = "core"
 }
 
+*/
+
+variable "existing_kubernetes_secret_name" {
+  type        = string
+  description = <<-EOT
+    If you are going to create the Kubernetes Secret the runner-controller will use
+    by some means (such as SOPS) outside of this component, set the name of the secret
+    here and it will be used. In this case, this component will not create a secret
+    and you can leave the secret-related inputs with their default (empty) values.
+    The same secret will be used by both the runner-controller and the webhook-server.
+    EOT
+  default     = ""
+}
+
 variable "s3_bucket_arns" {
   type        = list(string)
   description = "List of ARNs of S3 Buckets to which the runners will have read-write access to."
@@ -147,23 +162,30 @@ variable "runners" {
   EOT
 
   type = map(object({
-    type                           = string
-    scope                          = string
-    image                          = string
-    dind_enabled                   = bool
-    scale_down_delay_seconds       = number
-    min_replicas                   = number
-    max_replicas                   = number
-    busy_metrics                   = map(string)
+    type                     = string
+    scope                    = string
+    image                    = optional(string, "")
+    dind_enabled             = bool
+    scale_down_delay_seconds = number
+    min_replicas             = number
+    max_replicas             = number
+    busy_metrics = optional(object({
+      scale_up_threshold    = string
+      scale_down_threshold  = string
+      scale_up_adjustment   = optional(string)
+      scale_down_adjustment = optional(string)
+      scale_up_factor       = optional(string)
+      scale_down_factor     = optional(string)
+    }))
     webhook_driven_scaling_enabled = bool
     pull_driven_scaling_enabled    = bool
     labels                         = list(string)
-    storage                        = optional(string, false)
+    storage                        = optional(string, "")
     resources = object({
       limits = object({
         cpu               = string
         memory            = string
-        ephemeral_storage = optional(string, false)
+        ephemeral_storage = optional(string, "")
       })
       requests = object({
         cpu    = string
@@ -195,9 +217,21 @@ variable "eks_component_name" {
   default     = "eks/cluster"
 }
 
-variable "ssm_github_token_path" {
+variable "github_app_id" {
+  type        = string
+  description = "The ID of the GitHub App to use for the runner controller."
+  default     = ""
+}
+
+variable "github_app_installation_id" {
+  type        = string
+  description = "The \"Installation ID\" of the GitHub App to use for the runner controller."
+  default     = ""
+}
+
+variable "ssm_github_secret_path" {
   type        = string
-  description = "The path in SSM to the GitHub token."
+  description = "The path in SSM to the GitHub app private key file contents or GitHub PAT token."
   default     = ""
 }
 

src/variables.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.0"
+      version = ">= 4.9.0"
     }
     helm = {
       source  = "hashicorp/helm"