From 66831edeecbfb1a4ba97e88f1476d577f88a2fb6 Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Mon, 16 Jun 2025 18:55:24 +0200 Subject: [PATCH 1/4] Update SSM to use advanced tiering Due to the size of the JIT config returned by github we muse use advanced SSM config --- .../control-plane/src/scale-runners/scale-up.ts | 12 +++++++++--- lambdas/libs/aws-ssm-util/src/index.ts | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 08d16d682a..06c30faf7e 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -450,12 +450,18 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins metricGitHubAppRateLimit(runnerConfig.headers); // store jit config in ssm parameter store + // Must used advanced tier for SSM as encoded_jit_config can be > 4kb logger.debug('Runner JIT config for ephemeral runner generated.', { instance: instance, }); - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerConfig.data.encoded_jit_config, true, { - tags: [{ Key: 'InstanceId', Value: instance }], - }); + await putParameter( + `${githubRunnerConfig.ssmTokenPath}/${instance}`, + runnerConfig.data.encoded_jit_config, true, + { + tags: [{ Key: 'InstanceId', Value: instance }], + }, + "Advanced" + ); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit await delay(25); diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index a2d842b628..30bd1c07a5 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -22,6 +22,7 @@ export async function putParameter( parameter_value: string, secure: boolean, options: { tags?: Tag[] } = {}, + tier: "Advanced" | "Standard" = "Standard", ): Promise { const client = getTracedAWSV3Client(new SSMClient({ region: process.env.AWS_REGION })); await client.send( @@ -30,6 +31,7 @@ export async function putParameter( Value: parameter_value, Type: secure ? 'SecureString' : 'String', Tags: options.tags, + Tier: tier }), ); } From 1a207ed93142c04268abde0270d9ebb562e941fe Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Mon, 16 Jun 2025 18:58:37 +0200 Subject: [PATCH 2/4] fix formatting --- .../control-plane/src/scale-runners/scale-up.ts | 13 +++++++------ lambdas/libs/aws-ssm-util/src/index.ts | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 06c30faf7e..cdb4f551f6 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -455,12 +455,13 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins instance: instance, }); await putParameter( - `${githubRunnerConfig.ssmTokenPath}/${instance}`, - runnerConfig.data.encoded_jit_config, true, - { - tags: [{ Key: 'InstanceId', Value: instance }], - }, - "Advanced" + `${githubRunnerConfig.ssmTokenPath}/${instance}`, + runnerConfig.data.encoded_jit_config, + true, + { + tags: [{ Key: 'InstanceId', Value: instance }], + }, + 'Advanced', ); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index 30bd1c07a5..df2200336b 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -22,7 +22,7 @@ export async function putParameter( parameter_value: string, secure: boolean, options: { tags?: Tag[] } = {}, - tier: "Advanced" | "Standard" = "Standard", + tier: 'Advanced' | 'Standard' = 'Standard', ): Promise { const client = getTracedAWSV3Client(new SSMClient({ region: process.env.AWS_REGION })); await client.send( @@ -31,7 +31,7 @@ export async function putParameter( Value: parameter_value, Type: secure ? 'SecureString' : 'String', Tags: options.tags, - Tier: tier + Tier: tier, }), ); } From 49086caad34e8adc28725ecc8bff9294cabbcc65 Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Tue, 17 Jun 2025 13:49:39 +0200 Subject: [PATCH 3/4] Updates from code review --- docs/configuration.md | 6 ++++ .../src/scale-runners/scale-up.ts | 12 ++------ lambdas/libs/aws-ssm-util/src/index.test.ts | 28 ++++++++++++++++++- lambdas/libs/aws-ssm-util/src/index.ts | 9 ++++-- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bcf6d13aa1..98d3db6653 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -19,6 +19,12 @@ To be able to support a number of use-cases, the module has quite a lot of confi The module uses the AWS System Manager Parameter Store to store configuration for the runners, as well as registration tokens and secrets for the Lambdas. Paths for the parameters can be configured via the variable `ssm_paths`. The location of the configuration parameters is retrieved by the runners via the instance tag `ghr:ssm_config_path`. The following default paths will be used. Tokens or JIT config stored in the token path will be deleted after retrieval by instance, data not deleted after a day will be deleted by a SSM housekeeper lambda. +Furthermore, to accommodate larger JIT configurations or other stored values, the module implements automatic tier selection for SSM parameters: + +- **Parameter Tiering**: If the size of a parameter's value exceeds 4KB (specifically, 4000 bytes), the module will automatically use the 'Advanced' tier for that SSM parameter. Values smaller than this threshold will use the 'Standard' tier. +- **Cost Implications**: While the 'Standard' tier is generally free for a certain number of parameters and operations, the 'Advanced' tier incurs costs. These costs are typically pro-rated per hour for each parameter stored using the Advanced tier. For detailed and up-to-date pricing, please refer to the [AWS Systems Manager Pricing page](https://aws.amazon.com/systems-manager/pricing/#Parameter_Store). +- **Housekeeping Recommendation**: The last sentence of the "AWS SSM Parameters" section already mentions that "data not deleted after a day will be deleted by a SSM housekeeper lambda." It is crucial to ensure this or a similar housekeeping mechanism is active and correctly configured, especially considering the potential costs associated with 'Advanced' tier parameters. This utility should identify and delete any orphaned parameters to help manage costs and maintain a clean SSM environment. + | Path | Description | | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ssm_paths.root/var.prefix?/app/` | App secrets used by Lambda's | diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index cdb4f551f6..813df84af6 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -454,15 +454,9 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins logger.debug('Runner JIT config for ephemeral runner generated.', { instance: instance, }); - await putParameter( - `${githubRunnerConfig.ssmTokenPath}/${instance}`, - runnerConfig.data.encoded_jit_config, - true, - { - tags: [{ Key: 'InstanceId', Value: instance }], - }, - 'Advanced', - ); + await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerConfig.data.encoded_jit_config, true, { + tags: [{ Key: 'InstanceId', Value: instance }], + }); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit await delay(25); diff --git a/lambdas/libs/aws-ssm-util/src/index.test.ts b/lambdas/libs/aws-ssm-util/src/index.test.ts index 17c8424cf4..52e4242fdb 100644 --- a/lambdas/libs/aws-ssm-util/src/index.test.ts +++ b/lambdas/libs/aws-ssm-util/src/index.test.ts @@ -9,7 +9,7 @@ import 'aws-sdk-client-mock-jest/vitest'; import { mockClient } from 'aws-sdk-client-mock'; import nock from 'nock'; -import { getParameter, putParameter } from '.'; +import { getParameter, putParameter, SSM_ADVANCED_TIER_THRESHOLD } from '.'; import { describe, it, expect, beforeEach, vi } from 'vitest'; const mockSSMClient = mockClient(SSMClient); @@ -139,4 +139,30 @@ describe('Test getParameter and putParameter', () => { // Act await expect(getParameter(parameterName)).rejects.toThrow(`Parameter ${parameterName} not found`); }); + + it.each([ + ['a'.repeat(SSM_ADVANCED_TIER_THRESHOLD - 1), 'Standard'], + ['a'.repeat(SSM_ADVANCED_TIER_THRESHOLD), 'Advanced'], + ['a'.repeat(SSM_ADVANCED_TIER_THRESHOLD + 1), 'Advanced'], + ])('Puts parameters with value and sets correct SSM tier based on size and threshold', async (data, expectedTier) => { + // Arrange + const parameterValue = data; + const parameterName = 'testParamSmall'; + const secure = false; + const output: PutParameterCommandOutput = { + $metadata: { httpStatusCode: 200 }, + }; + mockSSMClient.on(PutParameterCommand).resolves(output); + + // Act + await putParameter(parameterName, parameterValue, secure); + + // Assert + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: parameterName, + Value: parameterValue, + Type: 'String', + Tier: expectedTier, + }); + }); }); diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index df2200336b..0b4925c17d 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -17,21 +17,26 @@ export async function getParameter(parameter_name: string): Promise { return result; } +export const SSM_ADVANCED_TIER_THRESHOLD = 4000; + export async function putParameter( parameter_name: string, parameter_value: string, secure: boolean, options: { tags?: Tag[] } = {}, - tier: 'Advanced' | 'Standard' = 'Standard', ): Promise { const client = getTracedAWSV3Client(new SSMClient({ region: process.env.AWS_REGION })); + + // Determine tier based on parameter_value size + const valueSizeBytes = Buffer.byteLength(parameter_value, 'utf8'); + await client.send( new PutParameterCommand({ Name: parameter_name, Value: parameter_value, Type: secure ? 'SecureString' : 'String', Tags: options.tags, - Tier: tier, + Tier: valueSizeBytes >= SSM_ADVANCED_TIER_THRESHOLD ? 'Advanced' : 'Standard', }), ); } From cee2ff92b2e7699fe93dcddbf64fa779cba68ddb Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Tue, 17 Jun 2025 14:31:56 +0200 Subject: [PATCH 4/4] Update lambdas/functions/control-plane/src/scale-runners/scale-up.ts --- lambdas/functions/control-plane/src/scale-runners/scale-up.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 813df84af6..08d16d682a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -450,7 +450,6 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins metricGitHubAppRateLimit(runnerConfig.headers); // store jit config in ssm parameter store - // Must used advanced tier for SSM as encoded_jit_config can be > 4kb logger.debug('Runner JIT config for ephemeral runner generated.', { instance: instance, });