Skip to content

Commit 02a8a68

Browse files
npalmbdruthbendavies
authored
Feature/idle runners (#113)
* Add support for ARM64 runners * Support Graviton (a1) and Graviton2 (*6g*) * Address TF format issues. * Address additional TF format error. * Add test case for arm64 release asset. * Add documentation changes for ARM64. * Make ARM/ICU patch conditional in user_data.sh. * Run pre-commit hooks. * Release 0.3.0 * Add feature to keep runners idle based on an idle config * Update modules/runners/variables.tf Co-authored-by: Ben Davies <[email protected]> * Update modules/runners/variables.tf Co-authored-by: Ben Davies <[email protected]> * Fix formatting * Fix unit test * Fix unit test * Implement idle config for terraform deployment * Fix logging * Set name for event rules * Add documentation for idle config * Add documentation for idle config * Add documentation for idle config Co-authored-by: Brice Ruth <[email protected]> Co-authored-by: Ben Davies <[email protected]>
1 parent 8171b41 commit 02a8a68

File tree

16 files changed

+467
-129
lines changed

16 files changed

+467
-129
lines changed

‎README.md

Lines changed: 87 additions & 39 deletions
Large diffs are not rendered by default.

‎examples/default/main.tf

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,13 @@ module "runners" {
3333
enable_organization_runners = false
3434
runner_extra_labels = "default,example"
3535

36-
# instance_type = "a1.large"
36+
# Uncommet idle config to have idle runners from 9 to 5 in time zone Amsterdam
37+
# idle_config = [{
38+
# cron = "* * 9-17 * * *"
39+
# timeZone = "Europe/Amsterdam"
40+
# idleCount = 1
41+
# }]
3742

3843
# disable KMS and encryption
39-
# encrypt_secrets = true
44+
# encrypt_secrets = false
4045
}

‎main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ module "runners" {
7474
runner_extra_labels = var.runner_extra_labels
7575
runner_as_root = var.runner_as_root
7676
runners_maximum_count = var.runners_maximum_count
77+
idle_config = var.idle_config
7778

7879
lambda_zip = var.runners_lambda_zip
7980
lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout
@@ -101,7 +102,6 @@ module "runner_binaries" {
101102
lambda_zip = var.runner_binaries_syncer_lambda_zip
102103
lambda_timeout = var.runner_binaries_syncer_lambda_timeout
103104

104-
105105
role_path = var.role_path
106106
role_permissions_boundary = var.role_permissions_boundary
107107
}

‎modules/runner-binaries-syncer/README.md

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,29 +34,39 @@ yarn run dist
3434
```
3535

3636
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
37+
## Requirements
38+
39+
No requirements.
40+
41+
## Providers
42+
43+
| Name | Version |
44+
|------|---------|
45+
| aws | n/a |
46+
3747
## Inputs
3848

3949
| Name | Description | Type | Default | Required |
40-
|------|-------------|:----:|:-----:|:-----:|
41-
| aws\_region | AWS region. | string | n/a | yes |
42-
| distribution\_bucket\_name | Bucket for storing the action runner distribution. | string | n/a | yes |
43-
| environment | A name that identifies the environment, used as prefix and for tagging. | string | n/a | yes |
44-
| lambda\_schedule\_expression | Scheduler expression for action runner binary syncer. | string | `"cron(27 * * * ? *)"` | no |
45-
| lambda\_timeout | Time out of the lambda in seconds. | number | `"300"` | no |
46-
| lambda\_zip | File location of the lambda zip file. | string | `"null"` | no |
47-
| role\_path | The path that will be added to the role, if not set the environment name will be used. | string | `"null"` | no |
48-
| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | string | `"null"` | no |
49-
| runner\_architecture | The platform architecture for the runner instance \(x64, arm64\), defaults to 'x64' | string | `"x64"` | no |
50-
| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | map(string) | `{}` | no |
50+
|------|-------------|------|---------|:--------:|
51+
| aws\_region | AWS region. | `string` | n/a | yes |
52+
| distribution\_bucket\_name | Bucket for storing the action runner distribution. | `string` | n/a | yes |
53+
| environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
54+
| lambda\_schedule\_expression | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no |
55+
| lambda\_timeout | Time out of the lambda in seconds. | `number` | `300` | no |
56+
| lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
57+
| role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
58+
| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
59+
| runner\_architecture | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
60+
| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
5161

5262
## Outputs
5363

5464
| Name | Description |
5565
|------|-------------|
56-
| bucket | |
57-
| lambda | |
58-
| lambda\_role | |
59-
| runner\_distribution\_object\_key | |
66+
| bucket | n/a |
67+
| lambda | n/a |
68+
| lambda\_role | n/a |
69+
| runner\_distribution\_object\_key | n/a |
6070

6171
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
6272

‎modules/runner-binaries-syncer/runner-binaries-syncer.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ resource "aws_iam_role_policy" "syncer" {
6060
}
6161

6262
resource "aws_cloudwatch_event_rule" "syncer" {
63+
name = "${var.environment}-syncer-rule"
6364
schedule_expression = var.lambda_schedule_expression
6465
tags = var.tags
6566
}

‎modules/runners/README.md

Lines changed: 49 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -48,52 +48,63 @@ yarn run dist
4848
```
4949

5050
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
51+
## Requirements
52+
53+
No requirements.
54+
55+
## Providers
56+
57+
| Name | Version |
58+
|------|---------|
59+
| aws | n/a |
60+
5161
## Inputs
5262

5363
| Name | Description | Type | Default | Required |
54-
|------|-------------|:----:|:-----:|:-----:|
55-
| ami\_filter | List of maps used to create the AMI filter for the action runner AMI. | map(list(string)) | `{ "name": [ "amzn2-ami-hvm-2.*-x86_64-ebs" ] }` | no |
56-
| ami\_owners | The list of owners used to select the AMI of action runner instances. | list(string) | `[ "amazon" ]` | no |
57-
| aws\_region | AWS region. | string | n/a | yes |
58-
| block\_device\_mappings | The EC2 instance block device configuration. Takes the following keys: `delete\_on\_termination`, `volume\_type`, `volume\_size`, `encrypted`, `iops` | map(string) | `{}` | no |
59-
| enable\_organization\_runners | | bool | n/a | yes |
60-
| encryption | KMS key to encrypted lambda environment secrets. Either provide a key and `encrypt` set to `true`. Or set the key to `null` and encrypt to `false`. | object | n/a | yes |
61-
| environment | A name that identifies the environment, used as prefix and for tagging. | string | n/a | yes |
62-
| github\_app | GitHub app parameters, see your github app. Ensure the key is base64 encoded. | object | n/a | yes |
63-
| instance\_profile\_path | The path that will be added to the instance\_profile, if not set the environment name will be used. | string | `"null"` | no |
64-
| instance\_type | Default instance type for the action runner. | string | `"m5.large"` | no |
65-
| lambda\_timeout\_scale\_down | Time out for the scale down lambda in seconds. | number | `"60"` | no |
66-
| lambda\_timeout\_scale\_up | Time out for the scale up lambda in seconds. | number | `"60"` | no |
67-
| lambda\_zip | File location of the lambda zip file. | string | `"null"` | no |
68-
| market\_options | Market options for the action runner instances. | string | `"spot"` | no |
69-
| minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | number | `"5"` | no |
70-
| overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name\_sg` overwrite the `Name` tag for all security groups created by this module. `name\_runner\_agent\_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name\_docker\_machine\_runners` override the `Name` tag spot instances created by the runner agent. | map(string) | `{ "name_runner": "", "name_sg": "" }` | no |
71-
| role\_path | The path that will be added to the role, if not set the environment name will be used. | string | `"null"` | no |
72-
| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | string | `"null"` | no |
73-
| runner\_architecture | The platform architecture of the runner instance\_type. | string | `"x64"` | no |
74-
| runner\_as\_root | Run the action runner under the root user. | bool | `"false"` | no |
75-
| runner\_extra\_labels | Extra labels for the runners \(GitHub\). Separate each label by a comma | string | `""` | no |
76-
| runners\_maximum\_count | The maximum number of runners that will be created. | number | `"3"` | no |
77-
| s3\_bucket\_runner\_binaries | | object | n/a | yes |
78-
| s3\_location\_runner\_binaries | S3 location of runner distribution. | string | n/a | yes |
79-
| scale\_down\_schedule\_expression | Scheduler expression to check every x for scale down. | string | `"cron(*/5 * * * ? *)"` | no |
80-
| sqs\_build\_queue | SQS queue to consume accepted build events. | object | n/a | yes |
81-
| subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc\_id`. | list(string) | n/a | yes |
82-
| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | map(string) | `{}` | no |
83-
| userdata\_post\_install | User-data script snippet to insert after GitHub acton runner install | string | `""` | no |
84-
| userdata\_pre\_install | User-data script snippet to insert before GitHub acton runner install | string | `""` | no |
85-
| vpc\_id | The VPC for the security groups. | string | n/a | yes |
64+
|------|-------------|------|---------|:--------:|
65+
| ami\_filter | List of maps used to create the AMI filter for the action runner AMI. | `map(list(string))` | <pre>{<br> "name": [<br> "amzn2-ami-hvm-2.*-x86_64-ebs"<br> ]<br>}</pre> | no |
66+
| ami\_owners | The list of owners used to select the AMI of action runner instances. | `list(string)` | <pre>[<br> "amazon"<br>]</pre> | no |
67+
| aws\_region | AWS region. | `string` | n/a | yes |
68+
| block\_device\_mappings | The EC2 instance block device configuration. Takes the following keys: `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops` | `map(string)` | `{}` | no |
69+
| enable\_organization\_runners | n/a | `bool` | n/a | yes |
70+
| encryption | KMS key to encrypted lambda environment secrets. Either provide a key and `encrypt` set to `true`. Or set the key to `null` and encrypt to `false`. | <pre>object({<br> kms_key_id = string<br> encrypt = bool<br> })</pre> | n/a | yes |
71+
| environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
72+
| github\_app | GitHub app parameters, see your github app. Ensure the key is base64 encoded. | <pre>object({<br> key_base64 = string<br> id = string<br> client_id = string<br> client_secret = string<br> })</pre> | n/a | yes |
73+
| idle\_config | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | <pre>list(object({<br> cron = string<br> timeZone = string<br> idleCount = number<br> }))</pre> | `[]` | no |
74+
| instance\_profile\_path | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no |
75+
| instance\_type | Default instance type for the action runner. | `string` | `"m5.large"` | no |
76+
| lambda\_timeout\_scale\_down | Time out for the scale down lambda in seconds. | `number` | `60` | no |
77+
| lambda\_timeout\_scale\_up | Time out for the scale up lambda in seconds. | `number` | `60` | no |
78+
| lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
79+
| market\_options | Market options for the action runner instances. | `string` | `"spot"` | no |
80+
| minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
81+
| overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` override the `Name` tag spot instances created by the runner agent. | `map(string)` | <pre>{<br> "name_runner": "",<br> "name_sg": ""<br>}</pre> | no |
82+
| role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
83+
| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
84+
| runner\_architecture | The platform architecture of the runner instance\_type. | `string` | `"x64"` | no |
85+
| runner\_as\_root | Run the action runner under the root user. | `bool` | `false` | no |
86+
| runner\_extra\_labels | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
87+
| runners\_maximum\_count | The maximum number of runners that will be created. | `number` | `3` | no |
88+
| s3\_bucket\_runner\_binaries | n/a | <pre>object({<br> arn = string<br> })</pre> | n/a | yes |
89+
| s3\_location\_runner\_binaries | S3 location of runner distribution. | `string` | n/a | yes |
90+
| scale\_down\_schedule\_expression | Scheduler expression to check every x for scale down. | `string` | `"cron(*/5 * * * ? *)"` | no |
91+
| sqs\_build\_queue | SQS queue to consume accepted build events. | <pre>object({<br> arn = string<br> })</pre> | n/a | yes |
92+
| subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes |
93+
| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
94+
| userdata\_post\_install | User-data script snippet to insert after GitHub acton runner install | `string` | `""` | no |
95+
| userdata\_pre\_install | User-data script snippet to insert before GitHub acton runner install | `string` | `""` | no |
96+
| vpc\_id | The VPC for the security groups. | `string` | n/a | yes |
8697

8798
## Outputs
8899

89100
| Name | Description |
90101
|------|-------------|
91-
| lambda\_scale\_down | |
92-
| lambda\_scale\_up | |
93-
| launch\_template | |
94-
| role\_runner | |
95-
| role\_scale\_down | |
96-
| role\_scale\_up | |
102+
| lambda\_scale\_down | n/a |
103+
| lambda\_scale\_up | n/a |
104+
| launch\_template | n/a |
105+
| role\_runner | n/a |
106+
| role\_scale\_down | n/a |
107+
| role\_scale\_up | n/a |
97108

98109
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
99110

‎modules/runners/lambdas/runners/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
"dependencies": {
2626
"@octokit/auth-app": "^2.4.11",
2727
"@octokit/rest": "^18.0.3",
28-
"moment": "^2.25.3",
28+
"cron-parser": "^2.15.0",
29+
"moment": "^2.27.0",
2930
"yn": "^4.0.0"
3031
}
3132
}

‎modules/runners/lambdas/runners/src/scale-runners/runners.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ export async function terminateRunner(runner: RunnerInfo): Promise<void> {
6363
InstanceIds: [runner.instanceId],
6464
})
6565
.promise();
66-
console.debug('Runner terminated.' + result.TerminatingInstances);
66+
console.debug('Runner terminated.' + runner.instanceId);
6767
}
6868

6969
export async function createRunner(runnerParameters: RunnerInputParameters): Promise<void> {

‎modules/runners/lambdas/runners/src/scale-runners/scale-down-config.test.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import moment from 'moment-timezone';
2+
import { getIdleRunnerCount, ScalingDownConfig, ScalingDownConfigList } from './scale-down-config';
3+
4+
const DEFAULT_TIMEZONE = 'America/Los_Angeles';
5+
const DEFAULT_IDLE_COUNT = 1;
6+
const now = moment.tz(new Date(), 'America/Los_Angeles');
7+
8+
function getConfig(cronTabs: string[]): ScalingDownConfigList {
9+
const result: ScalingDownConfigList = [];
10+
for (const cron of cronTabs) {
11+
result.push({
12+
cron: cron,
13+
idleCount: DEFAULT_IDLE_COUNT,
14+
timeZone: DEFAULT_TIMEZONE,
15+
});
16+
}
17+
return result;
18+
}
19+
20+
describe('scaleDownConfig', () => {
21+
beforeEach(() => {});
22+
23+
describe('Check runners that should be kept idle based on config.', () => {
24+
it('One active cron configuration', async () => {
25+
const scaleDownConfig = getConfig(['* * * * * *']);
26+
expect(getIdleRunnerCount(scaleDownConfig)).toEqual(DEFAULT_IDLE_COUNT);
27+
});
28+
29+
it('No active cron configuration', async () => {
30+
const scaleDownConfig = getConfig(['* * * * * ' + ((now.day() + 1) % 7)]);
31+
expect(getIdleRunnerCount(scaleDownConfig)).toEqual(0);
32+
});
33+
34+
it('1 of 2 cron configurations be active', async () => {
35+
const scaleDownConfig = getConfig(['* * * * * ' + ((now.day() + 1) % 7), '* * * * * ' + (now.day() % 7)]);
36+
expect(getIdleRunnerCount(scaleDownConfig)).toEqual(DEFAULT_IDLE_COUNT);
37+
});
38+
});
39+
});

‎modules/runners/lambdas/runners/src/scale-runners/scale-down-config.ts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import parser from 'cron-parser';
2+
import moment from 'moment';
3+
4+
export type ScalingDownConfigList = ScalingDownConfig[];
5+
export interface ScalingDownConfig {
6+
cron: string;
7+
idleCount: number;
8+
timeZone: string;
9+
}
10+
11+
function inPeriod(period: ScalingDownConfig): boolean {
12+
const now = moment(new Date());
13+
const expr = parser.parseExpression(period.cron, {
14+
tz: period.timeZone,
15+
});
16+
const next = moment(expr.next().toDate());
17+
return Math.abs(next.diff(now, 'seconds')) < 5; // we keep a range of 5 seconds
18+
}
19+
20+
export function getIdleRunnerCount(scalingDownConfigs: ScalingDownConfigList) {
21+
for (const scalingDownConfig of scalingDownConfigs) {
22+
if (inPeriod(scalingDownConfig)) return scalingDownConfig.idleCount;
23+
}
24+
return 0;
25+
}

0 commit comments

Comments
 (0)