feat(runners): add option to prefix registered runners in GitHub (#3043)

npalm · web-flow · commit ea4e0426cb32 · 2023-03-13T15:39:28.000+01:00
* feat: add support to prefix the runner

update start scripts

fix scripts

refactor names and manage fixed tags via terraform

* sync with main, and clenup

* update docs

* Only tag ec2

* set default prefix to empty string in case of an error

* Add separator in example for prefix
diff --git a/README.md b/README.md
@@ -524,6 +524,7 @@ We welcome any improvement to the standard module to make the default as secure
 | <a name="input_runner_iam_role_managed_policy_arns"></a> [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
 | <a name="input_runner_log_files"></a> [runner\_log\_files](#input\_runner\_log\_files) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | <pre>list(object({<br>    log_group_name   = string<br>    prefix_log_group = bool<br>    file_path        = string<br>    log_stream_name  = string<br>  }))</pre> | `null` | no |
 | <a name="input_runner_metadata_options"></a> [runner\_metadata\_options](#input\_runner\_metadata\_options) | Metadata options for the ec2 runner instances. By default, the module uses metadata tags for bootstrapping the runner, only disable `instance_metadata_tags` when using custom scripts for starting the runner. | `map(any)` | <pre>{<br>  "http_endpoint": "enabled",<br>  "http_put_response_hop_limit": 1,<br>  "http_tokens": "optional",<br>  "instance_metadata_tags": "enabled"<br>}</pre> | no |
+| <a name="input_runner_name_prefix"></a> [runner\_name\_prefix](#input\_runner\_name\_prefix) | The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is availabe via an EC2 tag 'ghr:runner\_name\_prefix'. | `string` | `""` | no |
 | <a name="input_runner_os"></a> [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no |
 | <a name="input_runner_run_as"></a> [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no |
 | <a name="input_runners_lambda_s3_key"></a> [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
diff --git a/examples/default/main.tf b/examples/default/main.tf
@@ -87,4 +87,7 @@ module "runners" {
   # enable_workflow_job_events_queue  = true
 
   enable_user_data_debug_logging_runner = true
+
+  # prefix GitHub runners with the environment name
+  runner_name_prefix = "${local.environment}_"
 }
diff --git a/examples/multi-runner/main.tf b/examples/multi-runner/main.tf
@@ -25,6 +25,7 @@ module "multi-runner" {
         runner_os                      = "linux"
         runner_architecture            = "arm64"
         runner_extra_labels            = "amazon"
+        runner_name_prefix             = "amazon-arm64_"
         enable_ssm_on_runners          = true
         instance_types                 = ["t4g.large", "c6g.large"]
         runners_maximum_count          = 1
@@ -47,6 +48,7 @@ module "multi-runner" {
         runner_architecture            = "x64"
         runner_extra_labels            = "ubuntu-latest,ubuntu-2204"
         runner_run_as                  = "ubuntu"
+        runner_name_prefix             = "ubuntu-2204-x64_"
         enable_ssm_on_runners          = true
         instance_types                 = ["m5ad.large", "m5a.large"]
         runners_maximum_count          = 1
@@ -101,6 +103,7 @@ module "multi-runner" {
       runner_config = {
         runner_os                      = "windows"
         runner_architecture            = "x64"
+        runner_name_prefix             = "servercore-2022-x64_"
         enable_ssm_on_runners          = true
         instance_types                 = ["m5.large", "c5.large"]
         runner_extra_labels            = "servercore-2022"
@@ -129,6 +132,7 @@ module "multi-runner" {
         }
         runner_os                       = "linux"
         runner_architecture             = "x64"
+        runner_name_prefix              = "amazon-x64_"
         create_service_linked_role_spot = true
         enable_ssm_on_runners           = true
         instance_types                  = ["m5ad.large", "m5a.large"]
diff --git a/main.tf b/main.tf
@@ -242,6 +242,7 @@ module "runners" {
   cloudwatch_config                = var.cloudwatch_config
   runner_log_files                 = var.runner_log_files
   runner_group_name                = var.runner_group_name
+  runner_name_prefix               = var.runner_name_prefix
 
   scale_up_reserved_concurrent_executions = var.scale_up_reserved_concurrent_executions
 
diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md
diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf
@@ -69,6 +69,7 @@ module "runners" {
   cloudwatch_config                = var.cloudwatch_config
   runner_log_files                 = each.value.runner_config.runner_log_files
   runner_group_name                = each.value.runner_config.runner_group_name
+  runner_name_prefix               = each.value.runner_config.runner_name_prefix
 
   scale_up_reserved_concurrent_executions = each.value.runner_config.scale_up_reserved_concurrent_executions
 
diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf
@@ -60,6 +60,7 @@ variable "multi_runner_config" {
       runner_boot_time_in_minutes             = optional(number, 5)
       runner_extra_labels                     = string
       runner_group_name                       = optional(string, "Default")
+      runner_name_prefix                      = optional(string, "")
       runner_run_as                           = optional(string, "ec2-user")
       runners_maximum_count                   = number
       scale_down_schedule_expression          = optional(string, "cron(*/5 * * * ? *)")
@@ -150,6 +151,7 @@ variable "multi_runner_config" {
         runner_boot_time_in_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
         runner_extra_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `enable_workflow_job_labels_check`. GitHub read-only labels should not be provided."
         runner_group_name: "Name of the runner group."
+        runner_name_prefix: "Prefix for the GitHub runner name."
         runner_run_as: "Run the GitHub actions agent as user."
         runners_maximum_count: "The maximum number of runners that will be created."
         scale_down_schedule_expression: "Scheduler expression to check every x for scale down."
diff --git a/modules/runners/README.md b/modules/runners/README.md
@@ -185,6 +185,7 @@ yarn run dist
 | <a name="input_runner_group_name"></a> [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no |
 | <a name="input_runner_iam_role_managed_policy_arns"></a> [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
 | <a name="input_runner_log_files"></a> [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/<var.prefix>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | <pre>list(object({<br>    log_group_name   = string<br>    prefix_log_group = bool<br>    file_path        = string<br>    log_stream_name  = string<br>  }))</pre> | `null` | no |
+| <a name="input_runner_name_prefix"></a> [runner\_name\_prefix](#input\_runner\_name\_prefix) | The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is availabe via an EC2 tag 'ghr:runner\_name\_prefix'. | `string` | `""` | no |
 | <a name="input_runner_os"></a> [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no |
 | <a name="input_runner_run_as"></a> [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no |
 | <a name="input_runners_lambda_s3_key"></a> [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
@@ -218,4 +219,5 @@ yarn run dist
 | <a name="output_role_runner"></a> [role\_runner](#output\_role\_runner) | n/a |
 | <a name="output_role_scale_down"></a> [role\_scale\_down](#output\_role\_scale\_down) | n/a |
 | <a name="output_role_scale_up"></a> [role\_scale\_up](#output\_role\_scale\_up) | n/a |
+| <a name="output_runners_log_groups"></a> [runners\_log\_groups](#output\_runners\_log\_groups) | List of log groups from different log files of runner machine. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/runners/lambdas/runners/src/aws/runners.test.ts b/modules/runners/lambdas/runners/src/aws/runners.test.ts
@@ -16,6 +16,7 @@ const ORG_NAME = 'SomeAwesomeCoder';
 const REPO_NAME = `${ORG_NAME}/some-amazing-library`;
 const ENVIRONMENT = 'unit-test-environment';
 const SSM_TOKEN_PATH = '/github-action-runners/default/runners/tokens';
+const RUNNER_NAME_PREFIX = '';
 
 const mockDescribeInstances = { promise: jest.fn() };
 mockEC2.describeInstances.mockImplementation(() => mockDescribeInstances);
@@ -28,6 +29,8 @@ const mockRunningInstances: AWS.EC2.DescribeInstancesResult = {
           InstanceId: 'i-1234',
           Tags: [
             { Key: 'ghr:Application', Value: 'github-action-runner' },
+            { Key: 'ghr:runner_name_prefix', Value: RUNNER_NAME_PREFIX },
+            { Key: 'ghr:created_by', Value: 'scale-up-lambda' },
             { Key: 'Type', Value: 'Org' },
             { Key: 'Owner', Value: 'CoderToCat' },
           ],
@@ -523,6 +526,7 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues):
         ResourceType: 'instance',
         Tags: [
           { Key: 'ghr:Application', Value: 'github-action-runner' },
+          { Key: 'ghr:created_by', Value: expectedValues.totalTargetCapacity > 1 ? 'pool-lambda' : 'scale-up-lambda' },
           { Key: 'Type', Value: expectedValues.type },
           { Key: 'Owner', Value: REPO_NAME },
         ],
diff --git a/modules/runners/lambdas/runners/src/aws/runners.ts b/modules/runners/lambdas/runners/src/aws/runners.ts
@@ -214,6 +214,7 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro
             ResourceType: 'instance',
             Tags: [
               { Key: 'ghr:Application', Value: 'github-action-runner' },
+              { Key: 'ghr:created_by', Value: numberOfRunners === 1 ? 'scale-up-lambda' : 'pool-lambda' },
               { Key: 'Type', Value: runnerParameters.runnerType },
               { Key: 'Owner', Value: runnerParameters.runnerOwner },
             ],
diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts
@@ -184,11 +184,11 @@ const DEFAULT_RUNNERS_ORIGINAL = [
 const DEFAULT_REGISTERED_RUNNERS = [
   {
     id: 101,
-    name: 'i-idle-101',
+    name: 'my-runner-i-idle-101',
   },
   {
     id: 102,
-    name: 'i-idle-102',
+    name: 'my-runner-i-idle-102',
   },
   {
     id: 103,
diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts
@@ -165,7 +165,7 @@ async function evaluateAndRemoveRunners(
     for (const ec2Runner of ec2RunnersFiltered) {
       const ghRunners = await listGitHubRunners(ec2Runner);
       const ghRunnersFiltered = ghRunners.filter((runner: { name: string }) =>
-        runner.name.startsWith(ec2Runner.instanceId),
+        runner.name.endsWith(ec2Runner.instanceId),
       );
       if (ghRunnersFiltered.length) {
         if (runnerMinimumTimeExceeded(ec2Runner)) {
diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts
@@ -175,6 +175,7 @@ describe('scaleUp with GHES', () => {
   describe('on org level', () => {
     beforeEach(() => {
       process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+      process.env.RUNNER_NAME_PREFIX = 'unit-test';
       expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS };
     });
 
@@ -251,6 +252,7 @@ describe('scaleUp with GHES', () => {
   describe('on repo level', () => {
     beforeEach(() => {
       process.env.ENABLE_ORGANIZATION_RUNNERS = 'false';
+      process.env.RUNNER_NAME_PREFIX = 'unit-test';
       expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS };
       expectedRunnerParams.runnerType = 'Repo';
       expectedRunnerParams.runnerOwner = `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`;
@@ -402,6 +404,7 @@ describe('scaleUp with public GH', () => {
   describe('on org level', () => {
     beforeEach(() => {
       process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+      process.env.RUNNER_NAME_PREFIX = 'unit-test';
       expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS };
       expectedRunnerParams.runnerServiceConfig = [
         `--url https://github.com/${TEST_DATA.repositoryOwner}`,
@@ -454,6 +457,7 @@ describe('scaleUp with public GH', () => {
   describe('on repo level', () => {
     beforeEach(() => {
       process.env.ENABLE_ORGANIZATION_RUNNERS = 'false';
+      process.env.RUNNER_NAME_PREFIX = 'unit-test';
       expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS };
       expectedRunnerParams.runnerType = 'Repo';
       expectedRunnerParams.runnerOwner = `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`;
diff --git a/modules/runners/main.tf b/modules/runners/main.tf
@@ -121,6 +121,9 @@ resource "aws_launch_template" "runner" {
       {
         "Name" = format("%s", local.name_runner)
       },
+      {
+        "ghr:runner_name_prefix" = var.runner_name_prefix
+      },
       var.runner_ec2_tags
     )
   }
@@ -132,6 +135,9 @@ resource "aws_launch_template" "runner" {
       {
         "Name" = format("%s", local.name_runner)
       },
+      {
+        "ghr:runner_name_prefix" = var.runner_name_prefix
+      },
       var.runner_ec2_tags
     )
   }
diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf
@@ -42,6 +42,7 @@ module "pool" {
       extra_labels              = var.runner_extra_labels
       launch_template           = aws_launch_template.runner
       group_name                = var.runner_group_name
+      name_prefix               = var.runner_name_prefix
       pool_owner                = var.pool_runner_owner
       role                      = aws_iam_role.runner
     }
diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf
@@ -34,6 +34,7 @@ resource "aws_lambda_function" "pool" {
       RUNNER_BOOT_TIME_IN_MINUTES          = var.config.runner.boot_time_in_minutes
       RUNNER_EXTRA_LABELS                  = var.config.runner.extra_labels
       RUNNER_GROUP_NAME                    = var.config.runner.group_name
+      RUNNER_NAME_PREFIX                   = var.config.runner.name_prefix
       RUNNER_OWNER                         = var.config.runner.pool_owner
       SSM_TOKEN_PATH                       = var.config.ssm_token_path
       SUBNET_IDS                           = join(",", var.config.subnet_ids)
diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf
@@ -34,8 +34,9 @@ variable "config" {
       launch_template = object({
         name = string
       })
-      group_name = string
-      pool_owner = string
+      group_name  = string
+      name_prefix = string
+      pool_owner  = string
       role = object({
         arn = string
       })
diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf
@@ -34,6 +34,7 @@ resource "aws_lambda_function" "scale_up" {
       PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name
       RUNNER_EXTRA_LABELS                  = lower(var.runner_extra_labels)
       RUNNER_GROUP_NAME                    = var.runner_group_name
+      RUNNER_NAME_PREFIX                   = var.runner_name_prefix
       RUNNERS_MAXIMUM_COUNT                = var.runners_maximum_count
       SSM_TOKEN_PATH                       = "${var.ssm_paths.root}/${var.ssm_paths.tokens}"
       SUBNET_IDS                           = join(",", var.subnet_ids)
diff --git a/modules/runners/templates/start-runner.ps1 b/modules/runners/templates/start-runner.ps1
@@ -20,6 +20,9 @@ Write-Host  "Retrieved tags from AWS API"
 $environment=$tags.Tags.where( {$_.Key -eq 'ghr:environment'}).value
 Write-Host  "Reteieved ghr:environment tag - ($environment)"
 
+$runner_name_prefix=$tags.Tags.where( {$_.Key -eq 'ghr:runner_name_prefix'}).value
+Write-Host  "Reteieved ghr:runner_name_prefix tag - ($runner_name_prefix)"
+
 $ssm_config_path=$tags.Tags.where( {$_.Key -eq 'ghr:ssm_config_path'}).value
 Write-Host  "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)"
 
@@ -91,7 +94,7 @@ foreach ($group in @("Administrators", "docker-users")) {
 Set-ItemProperty HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name ConsentPromptBehaviorAdmin -Value 0 -Force
 Write-Host "Disabled User Access Control (UAC)"
 
-$configCmd = ".\config.cmd --unattended --name $InstanceId --work `"_work`" $config"
+$configCmd = ".\config.cmd --unattended --name $runner_name_prefix$InstanceId --work `"_work`" $config"
 Write-Host "Configure GH Runner as user $run_as"
 Invoke-Expression $configCmd
 
diff --git a/modules/runners/templates/start-runner.sh b/modules/runners/templates/start-runner.sh
@@ -16,17 +16,21 @@ echo "Retrieved INSTANCE_ID from AWS API ($instance_id)"
 %{ if metadata_tags == "enabled" }
 environment=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:environment)
 ssm_config_path=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:ssm_config_path)
+runner_name_prefix=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:runner_name_prefix || echo "")
 
 %{ else }
 tags=$(aws ec2 describe-tags --region "$region" --filters "Name=resource-id,Values=$instance_id")
 echo "Retrieved tags from AWS API ($tags)"
 
 environment=$(echo "$tags" | jq -r '.Tags[]  | select(.Key == "ghr:environment") | .Value')
 ssm_config_path=$(echo "$tags" | jq -r '.Tags[]  | select(.Key == "ghr:ssm_config_path") | .Value')
+runner_name_prefix=$(echo "$tags" | jq -r '.Tags[]  | select(.Key == "ghr:runner_name_prefix") | .Value' || echo "")
+
 %{ endif }
 
 echo "Retrieved ghr:environment tag - ($environment)"
 echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)"
+echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)"
 
 parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}")
 echo "Retrieved parameters from AWS SSM ($parameters)"
@@ -74,7 +78,7 @@ fi
 chown -R $run_as .
 
 echo "Configure GH Runner as user $run_as"
-sudo --preserve-env=RUNNER_ALLOW_RUNASROOT -u "$run_as" -- ./config.sh --unattended --name "$instance_id" --work "_work" $${config}
+sudo --preserve-env=RUNNER_ALLOW_RUNASROOT -u "$run_as" -- ./config.sh --unattended --name "$runner_name_prefix$instance_id" --work "_work" $${config}
 
 info_arch=$(uname -p)
 info_os=$(( lsb_release -ds || cat /etc/*release || uname -om ) 2>/dev/null | head -n1 | cut -d "=" -f2- | tr -d '"')
diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf
@@ -582,3 +582,13 @@ variable "ssm_paths" {
     config = string
   })
 }
+
+variable "runner_name_prefix" {
+  description = "The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is availabe via an EC2 tag 'ghr:runner_name_prefix'."
+  type        = string
+  default     = ""
+  validation {
+    condition     = length(var.runner_name_prefix) <= 45
+    error_message = "The prefix used for the GitHub runner name must be less than 32 characters. AWS instances id are 17 chars, https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/resource-ids.html"
+  }
+}
diff --git a/variables.tf b/variables.tf
@@ -753,3 +753,13 @@ variable "ssm_paths" {
   })
   default = {}
 }
+
+variable "runner_name_prefix" {
+  description = "The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is availabe via an EC2 tag 'ghr:runner_name_prefix'."
+  type        = string
+  default     = ""
+  validation {
+    condition     = length(var.runner_name_prefix) <= 45
+    error_message = "The prefix used for the GitHub runner name must be less than 32 characters. AWS instances id are 17 chars, https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/resource-ids.html"
+  }
+}

Original file line number	Diff line number	Diff line change
`@@ -87,4 +87,7 @@ module "runners" {`
`87`	`87`	`# enable_workflow_job_events_queue = true`
`88`	`88`
`89`	`89`	`enable_user_data_debug_logging_runner = true`
	`90`	`+`
	`91`	`+ # prefix GitHub runners with the environment name`
	`92`	`+ runner_name_prefix = "${local.environment}_"`
`90`	`93`	`}`