VIA-615 AJ Move retries to different place

- add retry for fetchContentForVaccine()
- make log level as warn in fetchContentForVaccine() as it is retried from outside
- reduce timeout as API is retried

Author: ankur-jain-nhs

src/_lambda/content-cache-hydrator/content-fetcher.test.ts

@@ -23,7 +23,7 @@ describe("fetchContentForVaccine", () => {
     const actual = await fetchContentForVaccine(VaccineType.RSV);
     expect(axios.get).toHaveBeenCalledWith(
       `${testApiEndpoint}${CONTENT_API_PATH_PREFIX}${VaccineInfo[VaccineType.RSV].contentPath}`,
-      { headers: { accept: "application/json", apikey: testApiKey }, timeout: 30000 },
+      { headers: { accept: "application/json", apikey: testApiKey }, timeout: 10000 },
     );
     expect(actual).toBe(JSON.stringify(testApiContent));
   });

src/_lambda/content-cache-hydrator/content-fetcher.ts

@@ -20,23 +20,26 @@ const fetchContentForVaccine = async (vaccineType: VaccineType): Promise<string>
         accept: "application/json",
         apikey: apiKey,
       },
-      timeout: 30000,
+      timeout: 10000,
     });
     log.info({ context: { uri, vaccineType } }, "Successfully fetched content from API");
     return JSON.stringify(response.data);
   } catch (error) {
     if (error instanceof AxiosError) {
-      log.error({
-        error: {
-          code: error.code,
-          status: error.status,
-          message: error.message,
-          response_data: error.response?.data,
+      log.warn(
+        {
+          error: {
+            code: error.code,
+            status: error.status,
+            message: error.message,
+            response_data: error.response?.data,
+          },
+          context: { uri, vaccineType },
         },
-        context: { uri, vaccineType },
-      });
+        "AxiosError in getting vaccine content from nhs.uk API",
+      );
     } else {
-      log.error({ context: { uri, vaccineType } }, "Error in getting vaccine content from nhs.uk API");
+      log.warn({ context: { uri, vaccineType } }, "Error in getting vaccine content from nhs.uk API");
     }
     throw error;
   }

src/_lambda/content-cache-hydrator/handler.ts

@@ -49,10 +49,24 @@ async function hydrateCacheForVaccine(
   approvalEnabled: boolean,
   forceUpdate: boolean,
 ): Promise<HydrateCacheStatus> {
+  log.info({ context: { vaccineType } }, "Hydrating cache for given vaccine");
   const status: HydrateCacheStatus = { invalidatedCount: 0, failureCount: 0 };
 
+  const rateLimitDelayMillis: number = 1000 / ((await config.CONTENT_API_RATE_LIMIT_PER_MINUTE) / 60);
+  const rateLimitDelayWithMargin: number = 2 * rateLimitDelayMillis; // to keep ourselves well within the budget
+
   try {
-    const content: string = await fetchContentForVaccine(vaccineType);
+    const content: string = await retry(() => fetchContentForVaccine(vaccineType), {
+      retries: 2,
+      delay: (attempt: number) => {
+        const delayMillis = rateLimitDelayWithMargin * Math.pow(2, attempt + 1);
+        log.warn(
+          { context: { vaccineType, attempt, delayMillis } },
+          "Failed to fetch content for given vaccine, trying again",
+        );
+        return delayMillis;
+      },
+    });
     const filteredContent: VaccinePageContent = getFilteredContentForVaccine(vaccineType, content);
 
     if (!approvalEnabled) {
@@ -151,22 +165,18 @@ const runContentCacheHydrator = async (event: ContentCacheHydratorEvent) => {
   const rateLimitDelayMillis: number = 1000 / ((await config.CONTENT_API_RATE_LIMIT_PER_MINUTE) / 60);
   const rateLimitDelayWithMargin: number = 2 * rateLimitDelayMillis; // to keep ourselves well within the budget
   log.info(`Delay used between calls to rate limit content API is ${rateLimitDelayWithMargin}ms`);
+
   for (const vaccine of vaccinesToRunOn) {
-    const status = await retry(
-      async () => hydrateCacheForVaccine(vaccine, await config.CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED, forceUpdate),
-      {
-        retries: 3,
-        delay: (attempt) => {
-          const delayMillis = rateLimitDelayWithMargin * Math.pow(2, attempt);
-          log.warn({ context: { vaccine, attempt, delayMillis } }, "Failed to hydrate cache, trying again");
-          return delayMillis;
-        },
-      },
+    const status = await hydrateCacheForVaccine(
+      vaccine,
+      await config.CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED,
+      forceUpdate,
     );
 
     invalidatedCount += status.invalidatedCount;
     failureCount += status.failureCount;
-    await new Promise((f) => setTimeout(f, rateLimitDelayWithMargin)); // sleep
+
+    await new Promise((f) => setTimeout(f, rateLimitDelayWithMargin)); // sleep to rate limit
   }
 
   log.info({ context: { failureCount, invalidatedCount } }, "Finished hydrating content cache: report");