fix: allow disable JIT config for ephemeral runners (#3393)

* fix: allow disable JIT config for ephemeral runners

* docs: auto update terraform docs

* format

* review, stricter gates test coverage

* docs: auto update terraform docs

* fix: disable JIT by default

* fix: disable JIT by default

* fix: disable JIT by default

* docs: auto update terraform docs

* use app for generating docs

* docs: auto update terraform docs

* use app for generating docs

* use app for generating docs

* docs: auto update terraform docs

* use app for generating docs

* docs: auto update terraform docs

* enable jit config by default for ephemeral runners

* enable jit config by default for ephemeral runners

* docs: auto update terraform docs

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: npalm

.github/workflows/update-docs.yml

@@ -16,8 +16,28 @@ jobs:
     name: Auto update terraform docs
     runs-on: ubuntu-latest
     steps:
-      - if: github.event_name == 'push'
-        name: Checkout branch
+      - name: Get installation token
+        uses: philips-software/app-token-action@a37926571e4cec6f219e06727136efdd073d8657 # ratchet:philips-software/[email protected]
+        id: token
+        with:
+          app_id: ${{ secrets.FOREST_RELEASER_APP_ID }}
+          app_base64_private_key: ${{ secrets.FOREST_RELEASER_APP_PRIVATE_KEY_BASE64 }}
+          auth_type: installation
+
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+
+      # We use the app for branches in this this repo to ensure PR chekcs are kept in place.
+      - if: github.event_name == 'push' && github.repository_owner == 'philips-labs'
+        name: Checkout with App Token
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # ratchet:actions/checkout@v3
+        with:
+          token: ${{ steps.token.outputs.token }}
+
+      - if: github.event_name == 'push' && github.repository_owner != 'philips-labs'
+        name: Checkout with GITHUB Action token
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # ratchet:actions/checkout@v3
 
       - name: Generate TF docs
@@ -31,7 +51,7 @@ jobs:
         if: github.ref == 'refs/heads/main'
         uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # ratchet:peter-evans/[email protected]
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ steps.token.outputs.token || secrets.GITHUB_TOKEN }}
           commit-message: "Update Terraform docs"
           title: "docs: Update Terraform docs"
           branch: ${{ github.event.pull_request.base.ref }}-update-docs

README.md

@@ -92,7 +92,7 @@ To be able to support a number of use-cases the module has quite a lot of config
 - Multi-Runner module. This modules allows you to create multiple runner configurations with a single webhook and single GitHub App to simplify deployment of different types of runners. Refer to the [ReadMe](.modules/../modules/multi-runner/README.md) for more information to understand the functionality.
 - Workflow job event. You can configure the webhook in GitHub to send workflow job events to the webhook. Workflow job events were introduced by GitHub in September 2021 and are designed to support scalable runners. We advise using the workflow job event when possible.
 - Linux vs Windows. You can configure the OS types linux and win. Linux will be used by default.
-- Re-use vs Ephemeral. By default runners are re-used, until detected idle. Once idle they will be removed from the pool. To improve security we are introducing ephemeral runners. Those runners are only used for one job. Ephemeral runners are only working in combination with the workflow job event. For ephemeral runners the lambda requests a JIT (just in time) configuration object via the GitHub to register the runner. [JIT configuration](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-just-in-time-runners) is limited to ephemeral runners, for non ephemeral a registration token is requested. In both cases the configuration is made available to the instance via the same SSM parameter. We also suggest using a pre-build AMI to improve the start time of jobs.
+- Re-use vs Ephemeral. By default runners are re-used, until detected idle. Once idle they will be removed from the pool. To improve security we are introducing ephemeral runners. Those runners are only used for one job. Ephemeral runners are only working in combination with the workflow job event. For ephemeral runners the lambda requests a JIT (just in time) configuration via the GitHub API to register the runner. [JIT configuration](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-just-in-time-runners) is limited to ephemeral runners (and currently not supported by GHES). For non-ephemeral a registration token is requested always. In both cases the configuration is made available to the instance via the same SSM parameter. To disable JIT configuration for ephermeral runners set `enable_jit_config` to `false`. We also suggest using a pre-build AMI to improve the start time of jobs for ephemeral runners.
 - GitHub Cloud vs GitHub Enterprise Server (GHES). The runners support GitHub Cloud as well GitHub Enterprise Server. For GHES we rely on our community for support and testing. We have no possibility to test ourselves on GHES.
 - Spot vs on-demand. The runners use either the EC2 spot or on-demand life cycle. Runners will be created via the AWS [CreateFleet API](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html). The module (scale up lambda) will request via the CreateFleet API to create instances in one of the subnets and of the specified instance types.
 - ARM64 support via Graviton/Graviton2 instance-types. When using the default example or top-level module, specifying `instance_types` that match a Graviton/Graviton 2 (ARM64) architecture (e.g. a1, t4g or any 6th-gen `g` or `gd` type), you must also specify `runner_architecture = "arm64"` and the sub-modules will be automatically configured to provision with ARM64 AMIs and leverage GitHub's ARM64 action runner. See below for more details.
@@ -504,6 +504,7 @@ We welcome any improvement to the standard module to make the default as secure
 | <a name="input_enable_ephemeral_runners"></a> [enable\_ephemeral\_runners](#input\_enable\_ephemeral\_runners) | Enable ephemeral runners, runners will only be used once. | `bool` | `false` | no |
 | <a name="input_enable_event_rule_binaries_syncer"></a> [enable\_event\_rule\_binaries\_syncer](#input\_enable\_event\_rule\_binaries\_syncer) | Option to disable EventBridge Lambda trigger for the binary syncer, useful to stop automatic updates of binary distribution. | `bool` | `true` | no |
 | <a name="input_enable_fifo_build_queue"></a> [enable\_fifo\_build\_queue](#input\_enable\_fifo\_build\_queue) | Enable a FIFO queue to keep the order of events received by the webhook. Recommended for repo level runners. | `bool` | `false` | no |
+| <a name="input_enable_jit_config"></a> [enable\_jit\_config](#input\_enable\_jit\_config) | Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is avaialbe. In case you upgradeing from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI. | `bool` | `null` | no |
 | <a name="input_enable_job_queued_check"></a> [enable\_job\_queued\_check](#input\_enable\_job\_queued\_check) | Only scale if the job event received by the scale up lambda is in the queued state. By default enabled for non ephemeral runners and disabled for ephemeral. Set this variable to overwrite the default behavior. | `bool` | `null` | no |
 | <a name="input_enable_managed_runner_security_group"></a> [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enables creation of the default managed security group. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no |
 | <a name="input_enable_organization_runners"></a> [enable\_organization\_runners](#input\_enable\_organization\_runners) | Register runners to organization, instead of repo level | `bool` | `false` | no |

lambdas/functions/control-plane/jest.config.ts

@@ -6,9 +6,9 @@ const config: Config = {
   ...defaultConfig,
   coverageThreshold: {
     global: {
-      statements: 97,
-      branches: 93,
-      functions: 96,
+      statements: 97.6,
+      branches: 94.6,
+      functions: 97,
       lines: 98,
     },
   },

lambdas/functions/control-plane/src/pool/pool.ts

@@ -29,6 +29,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
   const instanceTypes = process.env.INSTANCE_TYPES.split(',');
   const instanceTargetTargetCapacityType = process.env.INSTANCE_TARGET_CAPACITY_TYPE;
   const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false });
+  const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral });
   const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false });
   const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME;
   const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE;
@@ -94,6 +95,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
     await createRunners(
       {
         ephemeral,
+        enableJitConfig,
         ghesBaseUrl,
         runnerLabels,
         runnerGroup,

lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts

@@ -592,6 +592,8 @@ describe('scaleUp with public GH', () => {
 
   describe('on repo level', () => {
     beforeEach(() => {
+      mockSSMClient.reset();
+
       process.env.ENABLE_ORGANIZATION_RUNNERS = 'false';
       process.env.RUNNER_NAME_PREFIX = 'unit-test';
       expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS };
@@ -668,12 +670,52 @@ describe('scaleUp with public GH', () => {
       ).rejects.toBeInstanceOf(Error);
     });
 
-    it('creates a ephemeral runner.', async () => {
+    it('creates a ephemeral runner with JIT config.', async () => {
       process.env.ENABLE_EPHEMERAL_RUNNERS = 'true';
       process.env.ENABLE_JOB_QUEUED_CHECK = 'false';
+      process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config';
+      await scaleUpModule.scaleUp('aws:sqs', TEST_DATA);
+      expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled();
+      expect(createRunner).toBeCalledWith(expectedRunnerParams);
+
+      expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, {
+        Name: '/github-action-runners/default/runners/config/i-12345',
+        Value: 'TEST_JIT_CONFIG_REPO',
+        Type: 'SecureString',
+      });
+    });
+
+    it('creates a ephemeral runner with registration token.', async () => {
+      process.env.ENABLE_EPHEMERAL_RUNNERS = 'true';
+      process.env.ENABLE_JIT_CONFIG = 'false';
+      process.env.ENABLE_JOB_QUEUED_CHECK = 'false';
+      process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config';
+      await scaleUpModule.scaleUp('aws:sqs', TEST_DATA);
+      expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled();
+      expect(createRunner).toBeCalledWith(expectedRunnerParams);
+
+      expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, {
+        Name: '/github-action-runners/default/runners/config/i-12345',
+        Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral',
+        Type: 'SecureString',
+      });
+    });
+
+    it('JIT config is ingored for non-ephemeral runners.', async () => {
+      process.env.ENABLE_EPHEMERAL_RUNNERS = 'false';
+      process.env.ENABLE_JIT_CONFIG = 'true';
+      process.env.ENABLE_JOB_QUEUED_CHECK = 'false';
+      process.env.RUNNER_LABELS = 'jit';
+      process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config';
       await scaleUpModule.scaleUp('aws:sqs', TEST_DATA);
       expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled();
       expect(createRunner).toBeCalledWith(expectedRunnerParams);
+
+      expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, {
+        Name: '/github-action-runners/default/runners/config/i-12345',
+        Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit',
+        Type: 'SecureString',
+      });
     });
 
     it('creates a ephemeral runner after checking job is queued.', async () => {

lambdas/functions/control-plane/src/scale-runners/scale-up.ts

@@ -32,6 +32,7 @@ export interface ActionRequestMessage {
 interface CreateGitHubRunnerConfig {
   ephemeral: boolean;
   ghesBaseUrl: string;
+  enableJitConfig: boolean;
   runnerLabels: string;
   runnerGroup: string;
   runnerNamePrefix: string;
@@ -57,8 +58,8 @@ function generateRunnerServiceConfig(githubRunnerConfig: CreateGitHubRunnerConfi
     `--token ${token}`,
   ];
 
-  if (githubRunnerConfig.runnerLabels !== undefined) {
-    config.push(`--labels ${githubRunnerConfig.runnerLabels}`);
+  if (githubRunnerConfig.runnerLabels) {
+    config.push(`--labels ${githubRunnerConfig.runnerLabels}`.trim());
   }
 
   if (githubRunnerConfig.disableAutoUpdate) {
@@ -69,6 +70,10 @@ function generateRunnerServiceConfig(githubRunnerConfig: CreateGitHubRunnerConfi
     config.push(`--runnergroup ${githubRunnerConfig.runnerGroup}`);
   }
 
+  if (githubRunnerConfig.ephemeral) {
+    config.push(`--ephemeral`);
+  }
+
   return config;
 }
 
@@ -221,6 +226,7 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
   const instanceTypes = process.env.INSTANCE_TYPES.split(',');
   const instanceTargetTargetCapacityType = process.env.INSTANCE_TARGET_CAPACITY_TYPE;
   const ephemeralEnabled = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false });
+  const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeralEnabled });
   const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false });
   const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME;
   const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE;
@@ -276,6 +282,7 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
       await createRunners(
         {
           ephemeral,
+          enableJitConfig,
           ghesBaseUrl,
           runnerLabels,
           runnerGroup,
@@ -313,10 +320,10 @@ async function createStartRunnerConfig(
   instances: string[],
   ghClient: Octokit,
 ) {
-  if (githubRunnerConfig.ephemeral) {
-    await createStartRunnerConfigForEphemeralRunners(githubRunnerConfig, instances, ghClient);
+  if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) {
+    await createJitConfig(githubRunnerConfig, instances, ghClient);
   } else {
-    await createStartRunnerConfigForNonEphemeralRunners(githubRunnerConfig, instances, ghClient);
+    await createRegistrationTokenConfig(githubRunnerConfig, instances, ghClient);
   }
 }
 
@@ -327,7 +334,7 @@ function addDelay(instances: string[]) {
   return { isDelay, delay };
 }
 
-async function createStartRunnerConfigForNonEphemeralRunners(
+async function createRegistrationTokenConfig(
   githubRunnerConfig: CreateGitHubRunnerConfig,
   instances: string[],
   ghClient: Octokit,
@@ -349,11 +356,7 @@ async function createStartRunnerConfigForNonEphemeralRunners(
   }
 }
 
-async function createStartRunnerConfigForEphemeralRunners(
-  githubRunnerConfig: CreateGitHubRunnerConfig,
-  instances: string[],
-  ghClient: Octokit,
-) {
+async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, instances: string[], ghClient: Octokit) {
   const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient);
   const { isDelay, delay } = addDelay(instances);
   const runnerLabels = githubRunnerConfig.runnerLabels.split(',');

main.tf

@@ -208,6 +208,7 @@ module "runners" {
   github_app_parameters                = local.github_app_parameters
   enable_organization_runners          = var.enable_organization_runners
   enable_ephemeral_runners             = var.enable_ephemeral_runners
+  enable_jit_config                    = var.enable_jit_config
   enable_job_queued_check              = var.enable_job_queued_check
   disable_runner_autoupdate            = var.disable_runner_autoupdate
   enable_managed_runner_security_group = var.enable_managed_runner_security_group