VIA-615 AJ Add logging and increase delay to avoid rate limits

Author: ankur-jain-nhs

src/_lambda/content-cache-hydrator/handler.ts

@@ -149,20 +149,24 @@ const runContentCacheHydrator = async (event: ContentCacheHydratorEvent) => {
   let invalidatedCount: number = 0;
 
   const rateLimitDelayMillis: number = 1000 / ((await config.CONTENT_API_RATE_LIMIT_PER_MINUTE) / 60);
-  const rateLimitDelayMarginFactor: number = 2; // to keep ourselves well within the budget
-  log.info(`Rate limit delay for content API is ${rateLimitDelayMillis}ms`);
+  const rateLimitDelayWithMargin: number = 2 * rateLimitDelayMillis; // to keep ourselves well within the budget
+  log.info(`Delay used between calls to rate limit content API is ${rateLimitDelayWithMargin}ms`);
   for (const vaccine of vaccinesToRunOn) {
     const status = await retry(
       async () => hydrateCacheForVaccine(vaccine, await config.CONTENT_CACHE_IS_CHANGE_APPROVAL_ENABLED, forceUpdate),
       {
         retries: 3,
-        delay: (attempt) => rateLimitDelayMillis * Math.pow(2, attempt - 1),
+        delay: (attempt) => {
+          const delayMillis = rateLimitDelayWithMargin * Math.pow(2, attempt);
+          log.warn({ context: { vaccine, attempt, delayMillis } }, "Failed to hydrate cache, trying again");
+          return delayMillis;
+        },
       },
     );
 
     invalidatedCount += status.invalidatedCount;
     failureCount += status.failureCount;
-    await new Promise((f) => setTimeout(f, rateLimitDelayMillis * rateLimitDelayMarginFactor)); // sleep
+    await new Promise((f) => setTimeout(f, rateLimitDelayWithMargin)); // sleep
   }
 
   log.info({ context: { failureCount, invalidatedCount } }, "Finished hydrating content cache: report");