Merge pull request #1477 from philips-labs/develop

release

Author: npalm

.ci/build.ps1

@@ -0,0 +1,8 @@
+$TOP_DIR=$(git rev-parse --show-toplevel)
+$OUTPUT_DIR="$TOP_DIR/lambda_output"
+
+New-Item "$OUTPUT_DIR" -ItemType Directory -ErrorAction SilentlyContinue
+
+$env:DOCKER_BUILDKIT=1 
+docker build --no-cache --target=final --output=type=local,dest="$OUTPUT_DIR" -f "$TOP_DIR/.ci/Dockerfile" "$TOP_DIR"
+

.editorconfig

@@ -0,0 +1,2 @@
+[*]
+end_of_line = lf

.github/workflows/packer-build.yml

@@ -0,0 +1,35 @@
+name: "Packer checks"
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+    paths:
+      - "images/**"
+      - ".github/workflows/packer-build.yml"
+
+env:  
+  AWS_REGION: eu-west-1
+
+jobs:
+  verify_packer:
+    name: Verify packer    
+    runs-on: ubuntu-latest
+    container:
+      image: hashicorp/packer:1.7.8
+    defaults:
+      run:
+        working-directory: images/linux-amzn2
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v2
+      
+      - name: packer init
+        run: packer init .
+
+      - name: check terraform formatting
+        run: packer fmt -recursive -check=true .        
+
+      - name: packer validate
+        run: packer validate .

.github/workflows/terraform.yml

@@ -43,7 +43,7 @@ jobs:
       fail-fast: false
       matrix:
         terraform: [0.14.3, 0.15.5, 1.0.8]
-        example: ["default", "ubuntu"]
+        example: ["default", "ubuntu", "prebuilt"]
     defaults:
       run:
         working-directory: examples/${{ matrix.example }}

README.md

@@ -16,6 +16,7 @@ This [Terraform](https://www.terraform.io/) module creates the required infrastr
     - [Install app](#install-app)
   - [Encryption](#encryption)
   - [Idle runners](#idle-runners)
+  - [Prebuilt Images](#prebuilt-images)
 - [Examples](#examples)
 - [Sub modules](#sub-modules)
   - [ARM64 configuration for submodules](#arm64-configuration-for-submodules)
@@ -265,6 +266,10 @@ idle_config = [{
 }]
 ```
 
+### Prebuilt Images
+
+This module also allows you to run agents from a prebuilt AMI to gain faster startup times. You can find more information in [the image README.md](/images/README.md)
+
 #### Supported config <!-- omit in toc -->
 
 Cron expressions are parsed by [cron-parser](https://github.com/harrisiirak/cron-parser#readme). The supported syntax.
@@ -289,6 +294,7 @@ Examples are located in the [examples](./examples) directory. The following exam
 
 - _[Default](examples/default/README.md)_: The default example of the module
 - _[Permissions boundary](examples/permissions-boundary/README.md)_: Example usages of permissions boundaries.
+- _[Prebuilt Images](examples/prebuilt/README.md)_: Example usages of deploying runners with a custom prebuilt image.
 
 ## Sub modules
 
@@ -346,10 +352,10 @@ In case the setup does not work as intended follow the trace of events:
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_runner_binaries"></a> [runner\_binaries](#module\_runner\_binaries) | ./modules/runner-binaries-syncer | n/a |
-| <a name="module_runners"></a> [runners](#module\_runners) | ./modules/runners | n/a |
-| <a name="module_ssm"></a> [ssm](#module\_ssm) | ./modules/ssm | n/a |
-| <a name="module_webhook"></a> [webhook](#module\_webhook) | ./modules/webhook | n/a |
+| <a name="module_runner_binaries"></a> [runner\_binaries](#module\_runner\_binaries) | ./modules/runner-binaries-syncer |  |
+| <a name="module_runners"></a> [runners](#module\_runners) | ./modules/runners |  |
+| <a name="module_ssm"></a> [ssm](#module\_ssm) | ./modules/ssm |  |
+| <a name="module_webhook"></a> [webhook](#module\_webhook) | ./modules/webhook |  |
 
 ## Resources
 
@@ -422,6 +428,7 @@ In case the setup does not work as intended follow the trace of events:
 | <a name="input_syncer_lambda_s3_key"></a> [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
 | <a name="input_syncer_lambda_s3_object_version"></a> [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
+| <a name="input_enabled_userdata"></a> [enabled_userdata](#input\_enabled_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no |
 | <a name="input_userdata_post_install"></a> [userdata\_post\_install](#input\_userdata\_post\_install) | Script to be ran after the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no |
 | <a name="input_userdata_pre_install"></a> [userdata\_pre\_install](#input\_userdata\_pre\_install) | Script to be ran before the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no |
 | <a name="input_userdata_template"></a> [userdata\_template](#input\_userdata\_template) | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no |

examples/prebuilt/.terraform.lock.hcl

@@ -0,0 +1,83 @@
+# Action runners deployment with prebuilt image
+
+This module shows how to create GitHub action runners using a prebuilt AMI for the runners
+
+## Usages
+
+Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](../../README.md). 
+
+### Lambdas
+
+You can either download the released lambda code or build them locally yourself.
+
+First download the Lambda releases from GitHub. Ensure you have set the version in `lambdas-download/main.tf` for running the example. The version needs to be set to a GitHub release version, see https://github.com/philips-labs/terraform-aws-github-runner/releases
+
+```bash
+cd lambdas-download
+terraform init
+terraform apply
+cd ..
+```
+
+Alternatively you can build the lambdas locally with Node or Docker, there is a simple build script in `<root>/.ci/build.sh`. In the `main.tf` you need to specify the build location for all of the zip files.
+
+```hcl
+  webhook_lambda_zip                = "../../lambda_output/webhook.zip"
+  runner_binaries_syncer_lambda_zip = "../../lambda_output/runner-binaries-syncer.zip"
+  runners_lambda_zip                = "../../lambda_output/runners.zip"
+```
+
+### GitHub App Configuration
+
+Before running Terraform, ensure the GitHub app is configured. See the [configuration details](../../README.md#usages) for more details.
+
+### Packer Image
+
+You will need to build your image. This example deployment uses the image example in `/images/linux-amz2`. You must build this image with packer in your AWS account first. Once you have built this you need to provider your owner ID as a variable
+
+## Deploy
+
+To use your image in the terraform modules you will need to set some values on the module.
+
+Assuming you have built the `linux-amzn2` image which has a pre-defined AMI name in the following format `github-runner-amzn2-x86_64-YYYYMMDDhhmm` you can use the following values.
+
+```hcl
+
+module "runners" {
+  ...
+  # set the name of the ami to use
+  ami_filter        = { name = ["github-runner-amzn2-x86_64-2021*"] }
+  # provide the owner id of 
+  ami_owners        = ["<your owner id>"]
+
+  enabled_userdata = false
+  ...
+}
+```
+
+If your owner is the same as the account you are logging into then you can use `aws_caller_identity` to retrieve it dynamically.
+
+```hcl
+data "aws_caller_identity" "current" {}
+
+module "runners" {
+  ...
+  ami_owners       = [data.aws_caller_identity.current.account_id]
+  ...
+}
+```
+
+You can then deploy the terraform
+
+```bash
+terraform init
+terraform apply
+```
+
+You can receive the webhook details by running:
+
+```bash
+terraform output -raw webhook_secret
+```
+
+Be-aware some shells will print some end of line character `%`.

examples/prebuilt/README.md

@@ -0,0 +1,25 @@
+locals {
+  version = "<REPLACE_BY_GITHUB_RELEASE_VERSION>"
+}
+
+module "lambdas" {
+  source = "../../../modules/download-lambda"
+  lambdas = [
+    {
+      name = "webhook"
+      tag  = local.version
+    },
+    {
+      name = "runners"
+      tag  = local.version
+    },
+    {
+      name = "runner-binaries-syncer"
+      tag  = local.version
+    }
+  ]
+}
+
+output "files" {
+  value = module.lambdas.files
+}

examples/prebuilt/lambdas-download/main.tf

@@ -0,0 +1,46 @@
+locals {
+  environment = "prebuilt"
+  aws_region  = "eu-west-1"
+}
+
+resource "random_password" "random" {
+  length = 28
+}
+
+data "aws_caller_identity" "current" {}
+
+module "runners" {
+  source                          = "../../"
+  create_service_linked_role_spot = true
+  aws_region                      = local.aws_region
+  vpc_id                          = module.vpc.vpc_id
+  subnet_ids                      = module.vpc.private_subnets
+
+  environment = local.environment
+
+  github_app = {
+    key_base64     = var.github_app_key_base64
+    id             = var.github_app_id
+    webhook_secret = random_password.random.result
+  }
+
+  webhook_lambda_zip                = "../../lambda_output/webhook.zip"
+  runner_binaries_syncer_lambda_zip = "../../lambda_output/runner-binaries-syncer.zip"
+  runners_lambda_zip                = "../../lambda_output/runners.zip"
+
+  runner_extra_labels = "default,example"
+
+  # configure your pre-built AMI
+  enabled_userdata = false
+  ami_filter       = { name = ["github-runner-amzn2-x86_64-2021*"] }
+  ami_owners       = [data.aws_caller_identity.current.account_id]
+
+  # enable access to the runners via SSM
+  enable_ssm_on_runners = true
+
+  # override delay of events in seconds
+  delay_webhook_event = 5
+
+  # override scaling down
+  scale_down_schedule_expression = "cron(* * * * ? *)"
+}

examples/prebuilt/main.tf

@@ -0,0 +1,15 @@
+output "runners" {
+  value = {
+    lambda_syncer_name = module.runners.binaries_syncer.lambda.function_name
+  }
+}
+
+output "webhook_endpoint" {
+  value = module.runners.webhook.endpoint
+}
+
+output "webhook_secret" {
+  sensitive = true
+  value     = random_password.random.result
+}
+