feat!: Add multi-runner capability (#2472)

Migration:
- Does the main module needs to be redeployed after this feature is released? There is no change in Terraform version for the main module, so it will not require TF upgrade. There is only one change in main module, which is a breaking change, the variable runner_enable_workflow_job_labels_check is not available under this module as we now only support the flagrunner_enable_workflow_job_labels_check_all to check all/partial workflow labels to be present in the runner labels. Running TF apply on existing deployment will apply some changes done in internal modules to existing infrastructure.
- Impact on using internal (submodules). There are some changes in the webhook internal module as per this feature. So, If you have been using it directly, the changes will be towards accepting runner configuration (queue details and label matchers) as a single object instead of earlier configuration of scattered queue details and criteria to match the workflow labels against the runner labels.

Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: navdeepg2021 <[email protected]>

Author: npalm

.github/workflows/terraform.yml

@@ -15,7 +15,7 @@ jobs:
     name: Verify module
     strategy:
       matrix:
-        terraform: [1.1.3, "latest"]
+        terraform: [1.3.2, "latest"]
     runs-on: ubuntu-latest
     container:
       image: hashicorp/terraform:${{ matrix.terraform }}
@@ -29,7 +29,7 @@ jobs:
           touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
       - name: terraform init
         run: terraform init -get -backend=false -input=false
-      - if: contains(matrix.terraform, '1.1.')
+      - if: contains(matrix.terraform, '1.3.')
         name: check terraform formatting
         run: terraform fmt -recursive -check=true -write=false
       - if: contains(matrix.terraform, 'latest') # check formatting for the latest release but avoid failing the build
@@ -44,7 +44,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        terraform: [1.0.11, 1.1.3, "latest"]
+        terraform: [1.0.11, 1.1.9, 1.2.9, "latest"]
         example:
           ["default", "ubuntu", "prebuilt", "arm64", "ephemeral", "windows"]
     defaults:
@@ -57,7 +57,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: terraform init
         run: terraform init -get -backend=false -input=false
-      - if: contains(matrix.terraform, '1.1.')
+      - if: contains(matrix.terraform, '1.3.')
         name: check terraform formatting
         run: terraform fmt -recursive -check=true -write=false
       - if: contains(matrix.terraform, 'latest') # check formatting for the latest release but avoid failing the build
@@ -66,3 +66,30 @@ jobs:
         continue-on-error: true
       - name: validate terraform011
         run: terraform validate
+
+
+  verify_multi_runner_example:
+    name: Verify Multi-Runner examples
+    strategy:
+      fail-fast: false
+      matrix:
+        terraform: [1.3.2, "latest"]
+    defaults:
+      run:
+        working-directory: examples/multi-runner
+    runs-on: ubuntu-latest
+    container:
+      image: hashicorp/terraform:${{ matrix.terraform }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: terraform init
+        run: terraform init -get -backend=false -input=false
+      - if: contains(matrix.terraform, '1.3.')
+        name: check terraform formatting
+        run: terraform fmt -recursive -check=true -write=false
+      - if: contains(matrix.terraform, 'latest') # check formatting for the latest release but avoid failing the build
+        name: check terraform formatting
+        run: terraform fmt -recursive -check=true -write=false
+        continue-on-error: true
+      - name: validate terraform
+        run: terraform validate

README.md

@@ -43,9 +43,6 @@ module "runners" {
   enable_organization_runners = true
   runner_extra_labels         = "default,example"
 
-  # enable workflow labels check
-  # runner_enable_workflow_job_labels_check = true
-
   # enable access to the runners via SSM
   enable_ssm_on_runners = true
 

examples/ephemeral/main.tf

@@ -0,0 +1,48 @@
+# Action runners deployment of Multiple-Runner-Configurations-Together example
+
+This module shows how to create GitHub action runners with multiple runner configuration together in one deployment. This example has the configurations for the following runner types with the relevant labels supported by them as matchers:
+
+- Linux ARM64 `["self-hosted", "linux", "arm64", "amazon"]`
+- Linux Ubuntu `["self-hosted", "linux", "x64", "ubuntu"]`
+- Linux X64 `["self-hosted", "linux", "x64", "amazon"]`
+- Windows X64 `["self-hosted", "windows", "x64", "servercore-2022"]`
+
+The module will decide the runner for the workflow job based on the match in the labels defined in the workflow job and runner configuration. Also the runner configuration allows the match to be exact or non-exact match. We recommend to use only exact matches.
+
+For exact match, all the labels defined in the workflow should be present in the runner configuration matchers and for non-exact match, some of the labels in the workflow, when present in runner configuration, shall be enough for the runner configuration to be used for the job. First the exact matchers are applied, next the non exact ones.
+
+## Webhook
+
+For the list of provided runner configurations, there will be a single webhook and only a single Github App to receive the notifications for all types of workflow triggers.
+
+## Lambda distribution
+
+Per combination of OS and architecture a lambda distribution syncer will be created. For this example there will be three instances (windows X64, linux X64, linux ARM).
+
+## Usages
+
+Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](../../README.md). First download the Lambda releases from GitHub. Alternatively you can build the lambdas locally with Node or Docker, there is a simple build script in `<root>/.ci/build.sh`. In the `main.tf` you can simply remove the location of the lambda zip files, the default location will work in this case.
+
+> Ensure you have set the version in `lambdas-download/main.tf` for running the example. The version needs to be set to a GitHub release version, see https://github.com/philips-labs/terraform-aws-github-runner/releases
+
+```bash
+cd lambdas-download
+terraform init
+terraform apply
+cd ..
+```
+
+Before running Terraform, ensure the GitHub app is configured. See the [configuration details](../../README.md#usages) for more details.
+
+```bash
+terraform init
+terraform apply
+```
+
+You can receive the webhook details by running:
+
+```bash
+terraform output -raw webhook_secret
+```
+
+Be-aware some shells will print some end of line character `%`.

examples/multi-runner/.terraform.lock.hcl

@@ -0,0 +1,25 @@
+locals {
+  version = "<REPLACE_BY_GITHUB_RELEASE_VERSION>"
+}
+
+module "lambdas" {
+  source = "../../../modules/download-lambda"
+  lambdas = [
+    {
+      name = "webhook"
+      tag  = local.version
+    },
+    {
+      name = "runners"
+      tag  = local.version
+    },
+    {
+      name = "runner-binaries-syncer"
+      tag  = local.version
+    }
+  ]
+}
+
+output "files" {
+  value = module.lambdas.files
+}

examples/multi-runner/README.md

@@ -0,0 +1,165 @@
+locals {
+  environment = var.environment != null ? var.environment : "multi-runner"
+  aws_region  = "eu-west-1"
+}
+
+resource "random_id" "random" {
+  byte_length = 20
+}
+data "aws_caller_identity" "current" {}
+
+
+################################################################################
+### Hybrid account
+################################################################################
+
+module "multi-runner" {
+  source = "../../modules/multi-runner"
+  multi_runner_config = {
+    "linux-arm64" = {
+      matcherConfig : {
+        labelMatchers = ["self-hosted", "linux", "arm64", "amazon"]
+        exactMatch    = true
+      }
+      fifo                = true
+      delay_webhook_event = 0
+      redrive_build_queue = {
+        enabled         = false
+        maxReceiveCount = null
+      }
+      runner_config = {
+        runner_os                      = "linux"
+        runner_architecture            = "arm64"
+        runner_extra_labels            = "arm"
+        enable_ssm_on_runners          = true
+        instance_types                 = ["t4g.large", "c6g.large"]
+        runners_maximum_count          = 1
+        scale_down_schedule_expression = "cron(* * * * ? *)"
+      }
+    },
+    "linux-ubuntu" = {
+      matcherConfig : {
+        labelMatchers = ["self-hosted", "linux", "x64", "ubuntu"]
+        exactMatch    = true
+      }
+      fifo                = true
+      delay_webhook_event = 0
+      redrive_build_queue = {
+        enabled         = false
+        maxReceiveCount = null
+      }
+      runner_config = {
+        runner_os                      = "linux"
+        runner_architecture            = "x64"
+        runner_extra_labels            = "ubuntu"
+        enable_ssm_on_runners          = true
+        instance_types                 = ["m5ad.large", "m5a.large"]
+        runners_maximum_count          = 1
+        scale_down_schedule_expression = "cron(* * * * ? *)"
+        runner_run_as                  = "ubuntu"
+        userdata_template              = "./templates/user-data.sh"
+        ami_owners                     = ["099720109477"] # Canonical's Amazon account ID
+
+        ami_filter = {
+          name = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+        }
+        block_device_mappings = [{
+          # Set the block device name for Ubuntu root device
+          device_name           = "/dev/sda1"
+          delete_on_termination = true
+          volume_type           = "gp3"
+          volume_size           = 30
+          encrypted             = true
+          iops                  = null
+          throughput            = null
+          kms_key_id            = null
+          snapshot_id           = null
+        }]
+        runner_log_files = [
+          {
+            log_group_name   = "syslog"
+            prefix_log_group = true
+            file_path        = "/var/log/syslog"
+            log_stream_name  = "{instance_id}"
+          },
+          {
+            log_group_name   = "user_data"
+            prefix_log_group = true
+            file_path        = "/var/log/user-data.log"
+            log_stream_name  = "{instance_id}/user_data"
+          },
+          {
+            log_group_name   = "runner"
+            prefix_log_group = true
+            file_path        = "/opt/actions-runner/_diag/Runner_**.log",
+            log_stream_name  = "{instance_id}/runner"
+          }
+        ]
+      }
+    },
+    "windows-x64" = {
+      matcherConfig : {
+        labelMatchers = ["self-hosted", "windows", "x64", "servercore-2022"]
+        exactMatch    = true
+      }
+      fifo                = true
+      delay_webhook_event = 5
+      runner_config = {
+        runner_os                      = "windows"
+        runner_architecture            = "x64"
+        enable_ssm_on_runners          = true
+        instance_types                 = ["m5.large", "c5.large"]
+        runner_extra_labels            = "servercore-2022"
+        runners_maximum_count          = 1
+        scale_down_schedule_expression = "cron(* * * * ? *)"
+        runner_boot_time_in_minutes    = 20
+        ami_filter = {
+          name = ["Windows_Server-2022-English-Core-ContainersLatest-*"]
+        }
+      }
+    },
+    "linux-x64" = {
+      matcherConfig : {
+        labelMatchers = ["self-hosted", "linux", "x64", "amazon"]
+        exactMatch    = false
+      }
+      fifo                = true
+      delay_webhook_event = 0
+      runner_config = {
+        runner_os                       = "linux"
+        runner_architecture             = "x64"
+        create_service_linked_role_spot = true
+        enable_ssm_on_runners           = true
+        instance_types                  = ["m5ad.large", "m5a.large"]
+        runner_extra_labels             = "amazon"
+        runners_maximum_count           = 1
+        enable_ephemeral_runners        = true
+        scale_down_schedule_expression  = "cron(* * * * ? *)"
+      }
+    }
+  }
+  aws_region                        = local.aws_region
+  vpc_id                            = module.vpc.vpc_id
+  subnet_ids                        = module.vpc.private_subnets
+  runners_scale_up_lambda_timeout   = 60
+  runners_scale_down_lambda_timeout = 60
+  prefix                            = local.environment
+  tags = {
+    Project = "ProjectX"
+  }
+  github_app = {
+    key_base64     = var.github_app_key_base64
+    id             = var.github_app_id
+    webhook_secret = random_id.random.hex
+  }
+
+  # Assuming local build lambda's to use pre build ones, uncomment the lines below and download the
+  # lambda zip files lambda_download
+  # webhook_lambda_zip                = "lambdas-download/webhook.zip"
+  # runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
+  # runners_lambda_zip                = "lambdas-download/runners.zip"
+
+  # override delay of events in seconds
+
+  # log_level = "debug"
+}

examples/multi-runner/lambdas-download/main.tf

@@ -0,0 +1,8 @@
+output "webhook_endpoint" {
+  value = module.multi-runner.webhook.endpoint
+}
+
+output "webhook_secret" {
+  sensitive = true
+  value     = random_id.random.hex
+}

examples/multi-runner/main.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = local.aws_region
+}