VIA-537 AS/SB Refactor content hydrator lambda, remove configProvider

Author: anoop-surej-nhs

src/_lambda/content-cache-hydrator/content-cache-reader.test.ts

@@ -5,9 +5,9 @@ import {
 import { VaccineInfo, VaccineType } from "@src/models/vaccine";
 import { readContentFromCache } from "@src/services/content-api/gateway/content-reader-service";
 import { InvalidatedCacheError, S3NoSuchKeyError } from "@src/services/content-api/gateway/exceptions";
-import { AppConfig, configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
+import { AsyncConfigMock, lazyConfigBuilder } from "@test-data/config/builders";
 
-jest.mock("@src/utils/config");
 jest.mock("@src/services/content-api/gateway/content-reader-service");
 jest.mock("sanitize-data", () => ({ sanitize: jest.fn() }));
 
@@ -16,13 +16,11 @@ const mockContentCachePath = "wiremock/__files/";
 describe("readCachedContentForVaccine", () => {
   const mockCacheFileContents = "mock-cache-file-contents";
   const vaccineType = VaccineType.RSV;
+  const mockedConfig = lazyConfig as AsyncConfigMock;
 
   beforeEach(() => {
-    (configProvider as jest.Mock).mockImplementation(
-      (): Partial<AppConfig> => ({
-        CONTENT_CACHE_PATH: mockContentCachePath,
-      }),
-    );
+    const defaultConfig = lazyConfigBuilder().withContentCachePath(mockContentCachePath).build();
+    Object.assign(mockedConfig, defaultConfig);
     (readContentFromCache as jest.Mock).mockImplementation((): string => mockCacheFileContents);
   });
 

src/_lambda/content-cache-hydrator/content-cache-reader.ts

@@ -1,7 +1,7 @@
 import { Filename, VaccineInfo, VaccineType } from "@src/models/vaccine";
 import { readContentFromCache } from "@src/services/content-api/gateway/content-reader-service";
 import { InvalidatedCacheError, S3NoSuchKeyError } from "@src/services/content-api/gateway/exceptions";
-import { AppConfig, configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
 import { logger } from "@src/utils/logger";
 
 const log = logger.child({ module: "content-cache-reader" });
@@ -12,12 +12,15 @@ export interface ReadCachedContentResult {
 }
 
 const readCachedContentForVaccine = async (vaccineType: VaccineType): Promise<ReadCachedContentResult> => {
-  const config: AppConfig = await configProvider();
   const cacheFilename: Filename = VaccineInfo[vaccineType].cacheFilename;
   let cachedContent: string;
 
   try {
-    cachedContent = await readContentFromCache(config.CONTENT_CACHE_PATH, cacheFilename, vaccineType);
+    cachedContent = await readContentFromCache(
+      (await lazyConfig.CONTENT_CACHE_PATH) as string,
+      cacheFilename,
+      vaccineType,
+    );
   } catch (error) {
     if (error instanceof S3NoSuchKeyError) {
       return { cacheStatus: "empty", cacheContent: "" };

src/_lambda/content-cache-hydrator/content-fetcher.test.ts

@@ -1,25 +1,24 @@
 import { CONTENT_API_PATH_PREFIX, fetchContentForVaccine } from "@src/_lambda/content-cache-hydrator/content-fetcher";
 import { VaccineInfo, VaccineType } from "@src/models/vaccine";
-import { AppConfig, configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
+import { AsyncConfigMock, lazyConfigBuilder } from "@test-data/config/builders";
 import axios from "axios";
 
-jest.mock("@src/utils/config");
 jest.mock("axios");
 jest.mock("sanitize-data", () => ({ sanitize: jest.fn() }));
 
 describe("fetchContentForVaccine", () => {
   const testApiKey: string = "test-key";
   const testApiEndpoint: URL = new URL("https://test-endpoint/");
   const testApiContent = { test: "content" };
+  const mockedConfig = lazyConfig as AsyncConfigMock;
 
   beforeEach(() => {
-    (configProvider as jest.Mock).mockImplementation(
-      (): Partial<AppConfig> => ({
-        CONTENT_CACHE_PATH: "",
-        CONTENT_API_KEY: testApiKey,
-        CONTENT_API_ENDPOINT: testApiEndpoint,
-      }),
-    );
+    const defaultConfig = lazyConfigBuilder()
+      .withContentApiKey(testApiKey)
+      .andContentApiEndpoint(testApiEndpoint)
+      .build();
+    Object.assign(mockedConfig, defaultConfig);
   });
 
   it("should fetch content for vaccine", async () => {

src/_lambda/content-cache-hydrator/content-fetcher.ts

@@ -1,17 +1,15 @@
 import { VaccineInfo, VaccineType } from "@src/models/vaccine";
-import { AppConfig, configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
 import { logger } from "@src/utils/logger";
 import axios, { AxiosError, AxiosResponse } from "axios";
 
 const log = logger.child({ module: "content-fetcher" });
 const CONTENT_API_PATH_PREFIX = "nhs-website-content/";
 
 const fetchContentForVaccine = async (vaccineType: VaccineType): Promise<string> => {
-  const config: AppConfig = await configProvider();
-
-  const apiEndpoint: URL = config.CONTENT_API_ENDPOINT;
+  const apiEndpoint: URL = (await lazyConfig.CONTENT_API_ENDPOINT) as URL;
   const vaccinePath = VaccineInfo[vaccineType].contentPath;
-  const apiKey: string = config.CONTENT_API_KEY;
+  const apiKey: string = (await lazyConfig.CONTENT_API_KEY) as string;
 
   const uri: string = `${apiEndpoint}${CONTENT_API_PATH_PREFIX}${vaccinePath}`;
   let response: AxiosResponse;

src/_lambda/content-cache-hydrator/content-writer-service.test.ts

@@ -8,10 +8,10 @@ import {
   writeContentForVaccine,
 } from "@src/_lambda/content-cache-hydrator/content-writer-service";
 import { Filename, VaccineInfo, VaccineType } from "@src/models/vaccine";
-import { configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
+import { AsyncConfigMock, lazyConfigBuilder } from "@test-data/config/builders";
 import { writeFile } from "node:fs/promises";
 
-jest.mock("@src/utils/config");
 jest.mock("node:fs/promises");
 
 let mockSend: jest.Mock = jest.fn();
@@ -29,9 +29,12 @@ describe("Content Writer Service", () => {
   const location: string = "test-location/";
   const path: Filename = "test-filename.json" as Filename;
   const content: string = "test-data";
+  const mockedConfig = lazyConfig as AsyncConfigMock;
 
   beforeEach(() => {
     mockSend = jest.fn();
+    const defaultConfig = lazyConfigBuilder().withContentCachePath(location).build();
+    Object.assign(mockedConfig, defaultConfig);
   });
 
   describe("_writeFileS3", () => {
@@ -74,10 +77,6 @@ describe("Content Writer Service", () => {
   });
 
   describe("writeContentForVaccine()", () => {
-    (configProvider as jest.Mock).mockImplementation(() => ({
-      CONTENT_CACHE_PATH: location,
-    }));
-
     it("should return response for rsv vaccine from content cache", async () => {
       const vaccine: VaccineType = VaccineType.RSV;
       await writeContentForVaccine(vaccine, content);

src/_lambda/content-cache-hydrator/content-writer-service.ts

@@ -1,7 +1,7 @@
 import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import { Filename, VaccineInfo, VaccineType } from "@src/models/vaccine";
-import { AppConfig, configProvider } from "@src/utils/config";
 import { AWS_PRIMARY_REGION } from "@src/utils/constants";
+import lazyConfig from "@src/utils/lazy-config";
 import { logger } from "@src/utils/logger";
 import { S3_PREFIX, isS3Path } from "@src/utils/path";
 import { writeFile } from "node:fs/promises";
@@ -38,9 +38,8 @@ const _writeContentToCache = async (
 };
 
 const writeContentForVaccine = async (vaccineType: VaccineType, vaccineContent: string) => {
-  const config: AppConfig = await configProvider();
   const cacheFilename = VaccineInfo[vaccineType].cacheFilename;
-  await _writeContentToCache(config.CONTENT_CACHE_PATH, cacheFilename, vaccineContent);
+  await _writeContentToCache((await lazyConfig.CONTENT_CACHE_PATH) as string, cacheFilename, vaccineContent);
 };
 
 export { _writeFileS3, _writeContentToCache, writeContentForVaccine };

src/_lambda/content-cache-hydrator/handler.test.ts

@@ -5,16 +5,16 @@ import {
 import { vitaContentChangedSinceLastApproved } from "@src/_lambda/content-cache-hydrator/content-change-detector";
 import { fetchContentForVaccine } from "@src/_lambda/content-cache-hydrator/content-fetcher";
 import { writeContentForVaccine } from "@src/_lambda/content-cache-hydrator/content-writer-service";
-import { _getConfigsThatThrowsOnColdStarts, handler } from "@src/_lambda/content-cache-hydrator/handler";
+import { handler } from "@src/_lambda/content-cache-hydrator/handler";
 import { invalidateCacheForVaccine } from "@src/_lambda/content-cache-hydrator/invalidate-cache";
 import { VaccineType } from "@src/models/vaccine";
 import { getFilteredContentForVaccine } from "@src/services/content-api/parsers/content-filter-service";
 import { getStyledContentForVaccine } from "@src/services/content-api/parsers/content-styling-service";
-import { configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
 import { RequestContext, asyncLocalStorage } from "@src/utils/requestContext";
+import { AsyncConfigMock, lazyConfigBuilder } from "@test-data/config/builders";
 import { Context } from "aws-lambda";
 
-jest.mock("@src/utils/config");
 jest.mock("@src/_lambda/content-cache-hydrator/content-writer-service");
 jest.mock("@src/_lambda/content-cache-hydrator/content-fetcher");
 jest.mock("@src/_lambda/content-cache-hydrator/content-cache-reader");
@@ -52,28 +52,12 @@ describe("Lambda Handler", () => {
     (writeContentForVaccine as jest.Mock).mockResolvedValue(undefined);
   });
 
-  describe("when config provider is flaky on cold starts", () => {
-    it("tries 3 times and throws when configProvider fails", async () => {
-      (configProvider as jest.Mock).mockRejectedValue(undefined);
-
-      await expect(_getConfigsThatThrowsOnColdStarts(0)).rejects.toThrow("Failed to get configs");
-
-      expect(configProvider).toHaveBeenCalledTimes(3);
-    });
-    it("returns configs when configProvider succeeds", async () => {
-      const testConfig = { test: "test" };
-      (configProvider as jest.Mock).mockResolvedValue(testConfig);
-
-      const configs = await _getConfigsThatThrowsOnColdStarts(0);
-      expect(configs).toBe(testConfig);
-
-      expect(configProvider).toHaveBeenCalledTimes(1);
-    });
-  });
-
   describe("when content-change-approval-needed feature disabled", () => {
+    const mockedConfig = lazyConfig as AsyncConfigMock;
+
     beforeEach(() => {
-      mockConfigProviderWithChangeApprovalSetTo(false);
+      const defaultConfig = lazyConfigBuilder().withContentCacheIsChangeApprovalEnabled(false).build();
+      Object.assign(mockedConfig, defaultConfig);
     });
 
     it("saves new vaccine content when cache was empty", async () => {
@@ -116,8 +100,11 @@ describe("Lambda Handler", () => {
   });
 
   describe("when content-change-approval-needed feature enabled", () => {
+    const mockedConfig = lazyConfig as AsyncConfigMock;
+
     beforeEach(() => {
-      mockConfigProviderWithChangeApprovalSetTo(true);
+      const defaultConfig = lazyConfigBuilder().withContentCacheIsChangeApprovalEnabled(true).build();
+      Object.assign(mockedConfig, defaultConfig);
     });
 
     it("overwrites invalidated cache with new updated content when forceUpdate is true in inbound event", async () => {
@@ -230,12 +217,6 @@ describe("Lambda Handler", () => {
     });
   });
 
-  const mockConfigProviderWithChangeApprovalSetTo = (changeApprovalEnabled: boolean) => {
-    (configProvider as jest.Mock).mockImplementation(() => ({
-      CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED: changeApprovalEnabled,
-    }));
-  };
-
   const mockReadCachedContentForVaccineWith = (mockInvalidatedCacheReadResult: ReadCachedContentResult) => {
     (readCachedContentForVaccine as jest.Mock).mockResolvedValue(mockInvalidatedCacheReadResult);
   };

src/_lambda/content-cache-hydrator/handler.ts

@@ -7,7 +7,7 @@ import { VaccineType } from "@src/models/vaccine";
 import { getFilteredContentForVaccine } from "@src/services/content-api/parsers/content-filter-service";
 import { getStyledContentForVaccine } from "@src/services/content-api/parsers/content-styling-service";
 import { VaccinePageContent } from "@src/services/content-api/types";
-import { AppConfig, configProvider } from "@src/utils/config";
+import lazyConfig from "@src/utils/lazy-config";
 import { logger } from "@src/utils/logger";
 import { getVaccineTypeFromLowercaseString } from "@src/utils/path";
 import { RequestContext, asyncLocalStorage } from "@src/utils/requestContext";
@@ -117,29 +117,6 @@ interface ContentCacheHydratorEvent {
   vaccineToUpdate?: string;
 }
 
-// currently whenever the cache hydrator lambda has a cold start (initialises)
-// the call to SSM fails with HTTP 400: Bad Request. It succeeds when lambda retries after 1min
-const _getConfigsThatThrowsOnColdStarts = async (waitBetweenTriesMillis: number = 5000): Promise<AppConfig> => {
-  const MaxTries: number = 3;
-  let tryCount: number = 1;
-
-  while (tryCount <= MaxTries) {
-    try {
-      return await configProvider();
-    } catch (error) {
-      log.warn(
-        {
-          error: error instanceof Error ? { message: error.message } : error,
-        },
-        `Failed to get configs. Sleeping for ${waitBetweenTriesMillis}ms and trying again. Tried ${tryCount}/${MaxTries}`,
-      );
-      await new Promise((resolve) => setTimeout(resolve, waitBetweenTriesMillis));
-    }
-    tryCount++;
-  }
-  throw new Error("Failed to get configs");
-};
-
 // Ref: https://nhsd-confluence.digital.nhs.uk/spaces/Vacc/pages/1113364124/Caching+strategy+for+content+from+NHS.uk+content+API
 const runContentCacheHydrator = async (event: ContentCacheHydratorEvent) => {
   log.info({ context: { event } }, "Received event, hydrating content cache.");
@@ -167,13 +144,15 @@ const runContentCacheHydrator = async (event: ContentCacheHydratorEvent) => {
     );
   }
 
-  const config: AppConfig = await _getConfigsThatThrowsOnColdStarts();
-
   let failureCount: number = 0;
   let invalidatedCount: number = 0;
 
   for (const vaccine of vaccinesToRunOn) {
-    const status = await hydrateCacheForVaccine(vaccine, config.CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED, forceUpdate);
+    const status = await hydrateCacheForVaccine(
+      vaccine,
+      (await lazyConfig.CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED) as boolean,
+      forceUpdate,
+    );
     invalidatedCount += status.invalidatedCount;
     failureCount += status.failureCount;
   }
@@ -195,5 +174,3 @@ export const handler = async (event: object, context: Context): Promise<void> =>
 
   await asyncLocalStorage.run(requestContext, () => runContentCacheHydrator(event));
 };
-
-export { _getConfigsThatThrowsOnColdStarts };

src/utils/lazy-config.ts

@@ -115,14 +115,18 @@ class LazyConfig {
 
   private async getFromEnvironmentOrSSM(key: string): Promise<string> {
     let value = process.env[key];
+    const initialDelayMillis = 100;
 
     if (value === undefined || value === null) {
       const ssmPrefix = await this.getSsmPrefix();
 
       log.debug({ context: { key, ssmPrefix } }, "getting from SSM");
+      // Get value from SSM, on failure retry won 100ms initially, with retry delays doubling each time
+      // 100ms -> 200ms -> 400ms etc
+      // Total ~ 100s
       value = await retry(() => getSSMParam(`${ssmPrefix}${key}`), {
         retries: 10,
-        delay: (attempt) => 100 * Math.pow(2, attempt - 1),
+        delay: (attempt) => initialDelayMillis * Math.pow(2, attempt - 1),
       });
     }
 

test-data/config/builders.tsx

@@ -120,6 +120,33 @@ class LazyConfigBuilder {
     return this;
   }
 
+  public withContentApiKey(value: string): this {
+    this._configValues.CONTENT_API_KEY = value;
+    return this;
+  }
+
+  public andContentApiKey(value: string): this {
+    return this.withContentApiKey(value);
+  }
+
+  public withContentApiEndpoint(value: URL): this {
+    this._configValues.CONTENT_API_ENDPOINT = value;
+    return this;
+  }
+
+  public andContentApiEndpoint(value: URL): this {
+    return this.withContentApiEndpoint(value);
+  }
+
+  public withContentCacheIsChangeApprovalEnabled(value: boolean): this {
+    this._configValues.CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED = value;
+    return this;
+  }
+
+  public andContentCacheIsChangeApprovalEnabled(value: boolean): this {
+    return this.withContentCacheIsChangeApprovalEnabled(value);
+  }
+
   public withEligibilityApiEndpoint(value: URL): this {
     this._configValues.ELIGIBILITY_API_ENDPOINT = value;
     return this;