VIA-614 AS Move SSM Parameters to Secrets Manager

anoop-surej-nhs · anoop-surej-nhs · commit 63a66ae2ae6c · 2025-11-28T10:11:32.000Z
diff --git a/.env.template b/.env.template
@@ -1,5 +1,5 @@
-# AWS parameter store
-SSM_PREFIX=/local/
+# AWS Secrets Manager
+SECRET_PREFIX=/local/
 
 # logging
 PINO_LOG_LEVEL=info
diff --git a/infrastructure/README.md b/infrastructure/README.md
@@ -4,22 +4,36 @@ Our infrastructure sits in the Europe(London) region coded 'eu-west-2' in AWS.
 
 ## Post deployment steps
 
-### Setting up and using secrets
+### Setting up and using secrets in AWS Secrets Manager
+
+Secrets need to be created in AWS Secrets Manager as follows:
+
+1. Click on `Store a new secret` in Secrets Manager
+2. Secret Type: `Other type of secret`
+3. Select `Plaintext` option
+4. Fill in the secret value in the text area
+5. Encryption key: `aws/secretsmanager`, and click next
+6. Add a name for the secret and description
+7. Tags as [below](#tags)
+8. Click next on the screens and then store.
 
 Update the values for the following secrets after generating them: -
 
-- /vita/apim/prod-1.pem - APIM private key used to sign JWTs generated from [here](https://digital.nhs.uk/developer/guides-and-documentation/security-and-authorisation/user-restricted-restful-apis-nhs-login-separate-authentication-and-authorisation#step-3-generate-a-key-pair). 'prod-1' here is the key id used during generation.
+- /vita/apim/prod-1.pem - APIM private key used to sign JWTs to access user-restricted APIs via APIM, generated from [here](https://digital.nhs.uk/developer/guides-and-documentation/security-and-authorisation/user-restricted-restful-apis-nhs-login-separate-authentication-and-authorisation#step-3-generate-a-key-pair). 'prod-1' here is the key id used during generation.
 - /vita/apim/prod-1.json - APIM public key in JWKS format generated above
 - /vita/nhslogin/private_key.pem - NHS Login private key generated from [here](https://nhsconnect.github.io/nhslogin/generating-pem/)
 - /vita/nhslogin/public_key.pem - NHS Login public key generated above
 - /vita/splunk/hec/endpoint - HEC endpoint of Splunk
 - /vita/splunk/hec/token - HEC token of Splunk endpoint to store operational logs
 
-Now fill the values used by the application below: -
+The following secrets need to be created and set before running the application:
 
-- Go to AWS service "Systems Manager"
-- Click on "Parameter Store" under application tools section
-- Update the values as per integrations in that environment
+- `/vita/APIM_PRIVATE_KEY`: Same value as `/vita/apim/prod-1.pem`
+- `/vita/AUTH_SECRET`: A randomly generated string used to sign JWTs for authentication (NextAuth)
+- `/vita/CONTENT_API_KEY`: NHS UK Content API Key
+- `/vita/ELIGIBILITY_API_KEY`: EliD API Key
+- `/vita/NHS_LOGIN_CLIENT_ID`: NHS Login OIDC Client ID for VitA App
+- `/vita/NHS_LOGIN_PRIVATE_KEY`: Same value as `/vita/nhslogin/private_key.pem`
 
 ### Setting up Cloudfront error pages
 
@@ -38,12 +52,10 @@ Manually create the following error routes.
 
 ### Setting default limits and settings
 
-- Increase the default throughput limit of the parameter store, instructions [here](https://docs.aws.amazon.com/systems-manager/latest/userguide/parameter-store-throughput.html#parameter-store-throughput-increasing)
 - Also setup [service quotas automatic management](https://docs.aws.amazon.com/servicequotas/latest/userguide/automatic-management.html) to alert on Slack channel
 - Review AWS > Service Quotas
   - AWS Lambda > Concurrent executions: Default is 1000 counts.
-  - AWS Systems Manager > Rate of GetParameter requests: Currently 10,000 per second. Cannot request an increase via AWS console.
-  - AWS Key Management Service (AWS KMS) > Cryptographic operations (symmetric) request rate: Currently 20,000 per second (used by SSM and Lambda)
+  - AWS Key Management Service (AWS KMS) > Cryptographic operations (symmetric) request rate: Currently 20,000 per second (used by Secrets Manager and Lambda)
 - Setup important notifications
   - AWS > User Notifications > Delivery Channels > Chat channels. Select the channel
   - Turn ON all 4 AWS managed subscriptions
diff --git a/infrastructure/environments/dev/locals.tf b/infrastructure/environments/dev/locals.tf
@@ -23,7 +23,7 @@ locals {
   splunk_log_retention_in_days = 7
 
   application_environment_variables = {
-    SSM_PREFIX = "/${local.prefix}/"
+    SECRET_PREFIX = "/${local.project_identifier_shortcode}/"
 
     PINO_LOG_LEVEL      = "info"
     DEPLOY_ENVIRONMENT  = local.environment
@@ -48,7 +48,7 @@ locals {
     NBS_URL          = "https://f.nhswebsite-integration.nhs.uk/nbs"
     NBS_BOOKING_PATH = "/nhs-app/vita"
 
-    SSM_PARAMETER_STORE_TTL                = 300
+    SECRETS_MANAGER_TTL                    = 300
     PARAMETERS_SECRETS_EXTENSION_LOG_LEVEL = "INFO"
   }
 
diff --git a/infrastructure/environments/dev/ssm.tf b/infrastructure/environments/dev/ssm.tf
@@ -1,3 +1,5 @@
+# TODO: Delete this file when we cut an R2 release
+# The scheduled assurances will fail if SSM parameters are deleted and redeployed whilst we still have R1
 resource "aws_ssm_parameter" "nhs_uk_content_api_key" {
   name             = "/${local.prefix}/CONTENT_API_KEY"
   type             = "SecureString"
diff --git a/infrastructure/environments/preprod/locals.tf b/infrastructure/environments/preprod/locals.tf
@@ -23,7 +23,7 @@ locals {
   splunk_log_retention_in_days = 30
 
   application_environment_variables = {
-    SSM_PREFIX = "/${local.prefix}/"
+    SECRET_PREFIX = "/${local.project_identifier_shortcode}/"
 
     PINO_LOG_LEVEL      = "info"
     DEPLOY_ENVIRONMENT  = local.environment
diff --git a/infrastructure/environments/preprod/ssm.tf b/infrastructure/environments/preprod/ssm.tf
@@ -1,3 +1,5 @@
+# TODO: Delete this file when we cut an R2 release
+# The scheduled assurances will fail if SSM parameters are deleted and redeployed whilst we still have R1
 resource "aws_ssm_parameter" "nhs_uk_content_api_key" {
   name             = "/${local.prefix}/CONTENT_API_KEY"
   type             = "SecureString"
diff --git a/infrastructure/environments/prod/locals.tf b/infrastructure/environments/prod/locals.tf
@@ -23,7 +23,7 @@ locals {
   splunk_log_retention_in_days = 90
 
   application_environment_variables = {
-    SSM_PREFIX = "/${local.prefix}/"
+    SECRET_PREFIX = "/${local.project_identifier_shortcode}/"
 
     PINO_LOG_LEVEL      = "info"
     DEPLOY_ENVIRONMENT  = local.environment
diff --git a/infrastructure/environments/prod/ssm.tf b/infrastructure/environments/prod/ssm.tf
@@ -1,3 +1,5 @@
+# TODO: Delete this file when we cut an R2 release
+# The scheduled assurances will fail if SSM parameters are deleted and redeployed whilst we still have R1
 resource "aws_ssm_parameter" "nhs_uk_content_api_key" {
   name             = "/${local.prefix}/CONTENT_API_KEY"
   type             = "SecureString"
diff --git a/infrastructure/environments/test/locals.tf b/infrastructure/environments/test/locals.tf
@@ -23,7 +23,7 @@ locals {
   splunk_log_retention_in_days = 30
 
   application_environment_variables = {
-    SSM_PREFIX = "/${local.prefix}/"
+    SECRET_PREFIX = "/${local.project_identifier_shortcode}/"
 
     PINO_LOG_LEVEL      = "debug"
     DEPLOY_ENVIRONMENT  = local.environment
diff --git a/infrastructure/environments/test/ssm.tf b/infrastructure/environments/test/ssm.tf
@@ -1,3 +1,5 @@
+# TODO: Delete this file when we cut an R2 release
+# The scheduled assurances will fail if SSM parameters are deleted and redeployed whilst we still have R1
 resource "aws_ssm_parameter" "nhs_uk_content_api_key" {
   name             = "/${local.prefix}/CONTENT_API_KEY"
   type             = "SecureString"
diff --git a/infrastructure/modules/deploy_app/policies/server-lambda-iam-policy.json b/infrastructure/modules/deploy_app/policies/server-lambda-iam-policy.json
@@ -8,13 +8,13 @@
     },
     {
       "Effect": "Allow",
-      "Action": ["ssm:GetParameter"],
-      "Resource": "arn:aws:ssm:${region}:${account_id}:parameter/${prefix}/*"
+      "Action": ["s3:ListBucket", "s3:GetObject"],
+      "Resource": "arn:aws:s3:::${prefix}*"
     },
     {
       "Effect": "Allow",
-      "Action": ["s3:ListBucket", "s3:GetObject"],
-      "Resource": "arn:aws:s3:::${prefix}*"
+      "Action": ["secretsmanager:GetSecretValue"],
+      "Resource": "arn:aws:secretsmanager:${region}:${account_id}:secret:/vita/*"
     }
   ]
 }
diff --git a/src/utils/config.test.ts b/src/utils/config.test.ts
@@ -1,8 +1,8 @@
 import config, { ConfigError } from "@src/utils/config";
-import getSSMParam from "@src/utils/get-ssm-param";
+import getSecret from "@src/utils/get-secret";
 import { randomString } from "@test-data/meta-builder";
 
-jest.mock("@src/utils/get-ssm-param");
+jest.mock("@src/utils/get-secret");
 jest.mock("sanitize-data", () => ({ sanitize: jest.fn() }));
 
 describe("lazyConfig", () => {
@@ -27,7 +27,7 @@ describe("lazyConfig", () => {
   });
 
   const setupTestEnvVars = (prefix: string) => {
-    process.env.SSM_PREFIX = prefix;
+    process.env.SECRET_PREFIX = prefix;
     process.env.CONTENT_API_ENDPOINT = "https://api-endpoint";
     process.env.ELIGIBILITY_API_ENDPOINT = "https://elid-endpoint";
     process.env.APIM_AUTH_URL = "https://apim-endpoint";
@@ -36,7 +36,7 @@ describe("lazyConfig", () => {
   it("should return values from env if present, else from getSSMParam", async () => {
     const prefix: string = "test/";
     setupTestEnvVars(prefix);
-    const mockGetSSMParam = (getSSMParam as jest.Mock).mockResolvedValue("api-key");
+    const mockGetSSMParam = (getSecret as jest.Mock).mockResolvedValue("api-key");
 
     expect(await config.CONTENT_API_ENDPOINT).toEqual(new URL("https://api-endpoint"));
     expect(await config.CONTENT_API_KEY).toEqual("api-key");
@@ -45,10 +45,10 @@ describe("lazyConfig", () => {
     expect(mockGetSSMParam).not.toHaveBeenCalledWith(`${prefix}CONTENT_API_ENDPOINT`);
   });
 
-  it("should throw error if values aren't in env or SSM", async () => {
+  it("should throw error if values aren't in env or Secrets Manager", async () => {
     const prefix: string = "test/";
-    process.env.SSM_PREFIX = prefix;
-    const mockGetSSMParam = (getSSMParam as jest.Mock).mockResolvedValue(undefined);
+    process.env.SECRET_PREFIX = prefix;
+    const mockGetSSMParam = (getSecret as jest.Mock).mockResolvedValue(undefined);
 
     await expect(async () => {
       await config.CONTENT_API_ENDPOINT;
@@ -106,7 +106,7 @@ describe("lazyConfig", () => {
 
   it("should reuse config values between subsequent calls", async () => {
     setupTestEnvVars("test/");
-    const mockGetSSMParam = (getSSMParam as jest.Mock).mockImplementation(() => randomString(5));
+    const mockGetSSMParam = (getSecret as jest.Mock).mockImplementation(() => randomString(5));
 
     await config.NHS_LOGIN_CLIENT_ID;
 
@@ -120,7 +120,7 @@ describe("lazyConfig", () => {
 
   it("should expire config after ttl", async () => {
     setupTestEnvVars("test/");
-    const mockGetSSMParam = (getSSMParam as jest.Mock).mockImplementation(() => "test-value");
+    const mockGetSSMParam = (getSecret as jest.Mock).mockImplementation(() => "test-value");
 
     await config.CONTENT_API_KEY;
 
@@ -133,15 +133,15 @@ describe("lazyConfig", () => {
     expect(mockGetSSMParam).toHaveBeenCalled();
   });
 
-  it("should retry fetching from SSM if the first attempt fails", async () => {
+  it("should retry fetching from Secrets Manager if the first attempt fails", async () => {
     const key = "CONTENT_API_KEY";
-    const expectedValue = "value-from-ssm-on-second-try";
+    const expectedValue = "value-from-secretsmanager-on-second-try";
     const expectedSsmPath = `/test/ci/${key}`;
 
-    process.env.SSM_PREFIX = "/test/ci/";
+    process.env.SECRET_PREFIX = "/test/ci/";
 
-    const mockGetSSMParam = (getSSMParam as jest.Mock)
-      .mockRejectedValueOnce(new Error("SSM is temporarily unavailable"))
+    const mockGetSSMParam = (getSecret as jest.Mock)
+      .mockRejectedValueOnce(new Error("SecretsManager is temporarily unavailable"))
       .mockResolvedValue(expectedValue);
 
     const resultPromise = config.CONTENT_API_KEY;
diff --git a/src/utils/config.ts b/src/utils/config.ts
@@ -1,5 +1,5 @@
 import { EligibilityApiError } from "@src/services/eligibility-api/gateway/exceptions";
-import getSSMParam from "@src/utils/get-ssm-param";
+import getSecret from "@src/utils/get-secret";
 import { logger } from "@src/utils/logger";
 import { retry } from "es-toolkit";
 import { Logger } from "pino";
@@ -9,7 +9,7 @@ const log: Logger = logger.child({ module: "lazy-config" });
 export type ConfigValue = string | number | boolean | URL | undefined;
 
 export interface AppConfig {
-  // SSM Params stored as SecureStrings
+  // SecretsManager secrets stored as SecureStrings
   NHS_LOGIN_CLIENT_ID: string;
   CONTENT_API_KEY: string;
   ELIGIBILITY_API_KEY: string;
@@ -65,7 +65,7 @@ function createReadOnlyDynamic<T extends object, C extends object>(instance: T):
 
 /**
  * Config object which only loads config items when, and crucially if, they are used.
- * Loads config from environment if it exists there, from SSM otherwise.
+ * Loads config from environment if it exists there, from SecretsManager otherwise.
  * Caches items for CACHE_TTL_MILLIS milliseconds, so we don't get items more than once.
  */
 class Config {
@@ -136,7 +136,7 @@ class Config {
     }
 
     log.debug({ context: { key } }, "cache miss");
-    const value = await this.getFromEnvironmentOrSSM(key);
+    const value = await this.getFromEnvironmentOrSecretsManager(key);
 
     const coercedValue = this._coerceType(key, value);
 
@@ -145,18 +145,18 @@ class Config {
     return coercedValue;
   }
 
-  private async getFromEnvironmentOrSSM(key: string): Promise<string> {
+  private async getFromEnvironmentOrSecretsManager(key: string): Promise<string> {
     let value = process.env[key];
     const initialDelayMillis = 100;
 
     if (value === undefined || value === null) {
-      const ssmPrefix = await this.getSsmPrefix();
+      const ssmPrefix = await this.getSecretPrefix();
 
-      log.debug({ context: { key, ssmPrefix } }, "getting from SSM");
-      // Get value from SSM, on failure retry won 100ms initially, with retry delays doubling each time
+      log.debug({ context: { key, ssmPrefix } }, "getting from SecretsManager");
+      // Get value from SecretsManager, on failure retry won 100ms initially, with retry delays doubling each time
       // 100ms -> 200ms -> 400ms etc
       // Total ~ 100s
-      value = await retry(() => getSSMParam(`${ssmPrefix}${key}`), {
+      value = await retry(() => getSecret(`${ssmPrefix}${key}`), {
         retries: 10,
         delay: (attempt) => initialDelayMillis * Math.pow(2, attempt - 1),
       });
@@ -170,21 +170,16 @@ class Config {
     return value;
   }
 
-  private async getSsmPrefix(): Promise<string> {
-    const key = "SSM_PREFIX";
-
-    if (this._cache.has(key)) {
-      return this._cache.get(key) as string;
-    }
+  private async getSecretPrefix(): Promise<string> {
+    const key = "SECRET_PREFIX";
 
     const prefix = process.env[key];
-    if (typeof prefix === "string" && prefix !== "") {
-      this._cache.set(key, prefix);
+    if (prefix) {
       return prefix;
     }
 
-    log.error({ context: { key } }, "SSM_PREFIX is not configured in the environment.");
-    throw new ConfigError("SSM_PREFIX is not configured correctly in the environment.");
+    log.error({ context: { key } }, "SECRET_PREFIX is not configured in the environment.");
+    throw new ConfigError("SECRET_PREFIX is not configured correctly in the environment.");
   }
 
   public resetCache() {
diff --git a/src/utils/get-secret.test.ts b/src/utils/get-secret.test.ts
@@ -0,0 +1,50 @@
+import getSecret from "@src/utils/get-secret";
+import axios from "axios";
+
+jest.mock("axios");
+jest.mock("sanitize-data", () => ({ sanitize: jest.fn() }));
+
+describe("getSSMParam", () => {
+  beforeEach(() => {
+    process.env.AWS_SESSION_TOKEN = "test-tok";
+  });
+
+  it("should call axios with the correct url", async () => {
+    const secretName = "anything";
+    const expectedValue = "test-value";
+    (axios.get as jest.Mock).mockResolvedValue({
+      data: { SecretString: expectedValue },
+    });
+
+    await getSecret(secretName);
+
+    expect(axios.get).toHaveBeenCalledWith(
+      expect.stringContaining("localhost:2773/secretsmanager/get"),
+      expect.objectContaining({
+        params: {
+          secretId: secretName,
+        },
+        headers: expect.objectContaining({
+          "X-Aws-Parameters-Secrets-Token": expect.any(String),
+        }),
+      }),
+    );
+  });
+
+  it("On success, returns correct parameter value", async () => {
+    const expectedValue = "test-value";
+    (axios.get as jest.Mock).mockResolvedValue({
+      data: { SecretString: expectedValue },
+    });
+
+    const actualValue: string = await getSecret("anything");
+
+    expect(actualValue).toBe(expectedValue);
+  });
+
+  it("should throw error when api call fails", () => {
+    (axios.get as jest.Mock).mockRejectedValue(new Error("error"));
+
+    expect(getSecret("anything")).rejects.toThrow("error");
+  });
+});
diff --git a/src/utils/get-secret.ts b/src/utils/get-secret.ts
diff --git a/src/utils/get-ssm-param.test.ts b/src/utils/get-ssm-param.test.ts

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+# TODO: Delete this file when we cut an R2 release`
	`2`	`+# The scheduled assurances will fail if SSM parameters are deleted and redeployed whilst we still have R1`
`1`	`3`	`resource "aws_ssm_parameter" "nhs_uk_content_api_key" {`
`2`	`4`	`name = "/${local.prefix}/CONTENT_API_KEY"`
`3`	`5`	`type = "SecureString"`