Sort out how we will expose issues reading the cache - important in future if we update the format

Lan2u · Lan2u · commit 7b537d295e6f · 2025-02-05T23:36:23.000Z
diff --git a/src/dependencies.ts b/src/dependencies.ts
@@ -1,13 +1,16 @@
 import {Logger} from 'pino';
 import {Failure, Email, DomainEvent, ResourceVersion} from './types';
 import * as TE from 'fp-ts/TaskEither';
+import * as t from 'io-ts';
 import {FailureWithStatus} from './types/failure-with-status';
 import {StatusCodes} from 'http-status-codes';
 
 import {Resource} from './types/resource';
 import {EventName, EventOfType} from './types/domain-event';
 import {SharedReadModel} from './read-models/shared-state';
 
+type TrainingSheetId = string;
+
 export type Dependencies = {
   commitEvent: (
     resource: Resource,
@@ -38,14 +41,22 @@ export type Dependencies = {
   sendEmail: (email: Email) => TE.TaskEither<Failure, string>;
   getCachedSheetData: () => TE.TaskEither<
     FailureWithStatus,
-    ReadonlyArray<
-      | EventOfType<'EquipmentTrainingQuizResult'>
-      | EventOfType<'EquipmentTrainingQuizSync'>
-    >
+    ReadonlyArray<{
+      cache_entry_id: string;
+      cached_timestamp: Date;
+      sheet_id: string;
+      equipment_id: string;
+      cached_data: t.Validation<
+        ReadonlyArray<
+          | EventOfType<'EquipmentTrainingQuizResult'>
+          | EventOfType<'EquipmentTrainingQuizSync'>
+        >
+      >;
+    }>
   >;
   cacheSheetData: (
     cacheTimestamp: Date,
-    sheetId: string,
+    sheetId: TrainingSheetId,
     data: ReadonlyArray<
       | EventOfType<'EquipmentTrainingQuizSync'>
       | EventOfType<'EquipmentTrainingQuizResult'>
diff --git a/src/index.ts b/src/index.ts
@@ -13,13 +13,13 @@ import {createTerminus} from '@godaddy/terminus';
 import http from 'http';
 import {pipe} from 'fp-ts/lib/function';
 import * as TE from 'fp-ts/TaskEither';
-import * as E from 'fp-ts/Either';
 import {ensureEventTableExists} from './init-dependencies/event-store/ensure-events-table-exists';
 import {initDependencies} from './init-dependencies';
 import * as libsqlClient from '@libsql/client';
 import cookieSession from 'cookie-session';
 import {initRoutes} from './routes';
 import {ensureCachedSheetDataTableExists} from './init-dependencies/google/ensure-cached-sheet-data-table-exists';
+import {loadCachedSheetData} from './load-cached-sheet-data';
 
 // Dependencies and Config
 const conf = loadConfig();
@@ -95,21 +95,12 @@ void (async () => {
   )();
 
   deps.logger.info('Loading cached external events...');
-  const events = await deps.getCachedSheetData()();
-  if (E.isLeft(events)) {
-    // Potential pitfall here - transient db errors could produce large spikes in processing.
-    // Tradeoff is that an error/bug in cached sheet data doesn't bring down the application.
-    deps.logger.error(
-      events.left,
-      'Failed to load cached external events data - continuing anyway...'
-    );
-  } else {
-    deps.logger.info(
-      'Loaded %s events from external events data cache',
-      events.right.length
-    );
-    events.right.forEach(deps.sharedReadModel.updateState);
-  }
+  await loadCachedSheetData(
+    deps.getCachedSheetData,
+    deps.logger,
+    deps.sharedReadModel.updateState
+  );
+  deps.logger.info('Cached external events loaded');
   await deps.sharedReadModel.asyncRefresh()();
   await deps.sharedReadModel.asyncApplyExternalEventSources()();
 
diff --git a/src/init-dependencies/google/get-cached-sheet-data.ts b/src/init-dependencies/google/get-cached-sheet-data.ts
@@ -60,9 +60,9 @@ export const getCachedSheetData =
       TE.map(table =>
         pipe(
           table.rows,
-          RA.map(entry => ({
-            ...entry,
-            cached_data: extractCachedEvents(entry.cached_data),
+          RA.map(row => ({
+            ...row,
+            cached_data: extractCachedEvents(row.cached_data),
           }))
         )
       )
diff --git a/src/load-cached-sheet-data.ts b/src/load-cached-sheet-data.ts
@@ -0,0 +1,44 @@
+import {Logger} from 'pino';
+import * as E from 'fp-ts/Either';
+import {Dependencies} from './dependencies';
+import {SharedReadModel} from './read-models/shared-state';
+
+export const loadCachedSheetData = async (
+  getCachedSheetData: Dependencies['getCachedSheetData'],
+  logger: Logger,
+  updateState: SharedReadModel['updateState']
+) => {
+  const events = await getCachedSheetData()();
+  if (E.isLeft(events)) {
+    // Potential pitfall here - transient db errors could produce large spikes in processing.
+    // Tradeoff is that an error/bug in cached sheet data doesn't bring down the application.
+    logger.error(
+      events.left,
+      'Failed to load any cached external events data - continuing anyway...'
+    );
+  } else {
+    logger.info(
+      'Loaded events for %s pieces of equipment from external events data cache',
+      events.right.length
+    );
+    events.right.forEach(cachedSheetData => {
+      if (E.isLeft(cachedSheetData.cached_data)) {
+        logger.info(
+          'Failed to load events for sheet %s, equipment %s generated at %s, skipping...',
+          cachedSheetData.sheet_id,
+          cachedSheetData.equipment_id,
+          cachedSheetData.cached_timestamp.toISOString()
+        );
+      } else {
+        logger.info(
+          'Loaded % events for sheet %s, equipment %s generated at %s',
+          cachedSheetData.cached_data.right.length,
+          cachedSheetData.sheet_id,
+          cachedSheetData.equipment_id,
+          cachedSheetData.cached_timestamp.toISOString()
+        );
+        cachedSheetData.cached_data.right.forEach(updateState);
+      }
+    });
+  }
+};
diff --git a/tests/read-models/test-framework.ts b/tests/read-models/test-framework.ts
@@ -19,6 +19,8 @@ import {Dependencies} from '../../src/dependencies';
 import {applyToResource} from '../../src/commands/apply-command-to-resource';
 import {initSharedReadModel} from '../../src/read-models/shared-state';
 import {localGoogleHelpers} from '../init-dependencies/pull-local-google';
+import {cacheSheetData} from '../../src/init-dependencies/google/get-cached-sheet-data';
+import {ensureCachedSheetDataTableExists} from '../../src/init-dependencies/google/ensure-cached-sheet-data-table-exists';
 
 type ToFrameworkCommands<T> = {
   [K in keyof T]: {
@@ -58,14 +60,16 @@ export const initTestFramework = async (
     dbClient,
     logger,
     O.some(localGoogleHelpers),
-    googleRateLimitMs
+    googleRateLimitMs,
+    cacheSheetData(dbClient)
   );
   const frameworkCommitEvent = commitEvent(
     dbClient,
     logger,
     sharedReadModel.asyncRefresh
   );
   await ensureEventTableExists(dbClient)();
+  await ensureCachedSheetDataTableExists(dbClient)();
   const frameworkGetAllEvents = () =>
     pipe(getAllEvents(dbClient)(), T.map(getRightOrFail))();
   const frameworkGetAllEventsByType = <EN extends EventName>(eventType: EN) =>
diff --git a/tests/training-sheets/cached-sheet-data.test.ts b/tests/training-sheets/cached-sheet-data.test.ts
@@ -5,6 +5,7 @@ import {
   getCachedSheetData,
 } from '../../src/init-dependencies/google/get-cached-sheet-data';
 import {UUID} from 'io-ts-types';
+import * as t from 'io-ts';
 import {faker} from '@faker-js/faker';
 import {constructEvent, EventOfType} from '../../src/types/domain-event';
 import {getRightOrFail} from '../helpers';
@@ -14,14 +15,23 @@ describe('Cache sheet data', () => {
   describe('Cache then restore', () => {
     const equipmentId = '326e8fda-7be8-4cd8-87d5-7cdfafebf996' as UUID;
     const sheetId = 'myTestingSheetId';
-    let cachedData: ReadonlyArray<
-      | EventOfType<'EquipmentTrainingQuizResult'>
-      | EventOfType<'EquipmentTrainingQuizSync'>
-    >;
+    const cacheTimestamp = new Date(2024, 1, 23, 4, 23, 45);
+    let cachedData: ReadonlyArray<{
+      cache_entry_id: string;
+      cached_timestamp: Date;
+      sheet_id: string;
+      equipment_id: string;
+      cached_data: t.Validation<
+        ReadonlyArray<
+          | EventOfType<'EquipmentTrainingQuizResult'>
+          | EventOfType<'EquipmentTrainingQuizSync'>
+        >
+      >;
+    }>;
     beforeEach(async () => {
       const db = libsqlClient.createClient({url: ':memory:'});
       await ensureCachedSheetDataTableExists(db)();
-      await cacheSheetData(db)(new Date(2024, 1, 23, 4, 23, 45), sheetId, [
+      await cacheSheetData(db)(cacheTimestamp, sheetId, [
         constructEvent('EquipmentTrainingQuizSync')({
           equipmentId,
         }),
@@ -39,8 +49,16 @@ describe('Cache sheet data', () => {
       ])();
       cachedData = getRightOrFail(await getCachedSheetData(db)()());
     });
+    it('Each sheet is cached', () => {
+      expect(cachedData).toHaveLength(1); // 1 sheet
+    });
     it('All events cached are returned', () => {
-      expect(cachedData).toHaveLength(2);
+      expect(getRightOrFail(cachedData[0].cached_data)).toHaveLength(2); // 2 events.
+    });
+    it('Event cache is correctly labeled', () => {
+      expect(cachedData[0].equipment_id).toStrictEqual(equipmentId);
+      expect(cachedData[0].sheet_id).toStrictEqual(sheetId);
+      expect(cachedData[0].cached_timestamp).toStrictEqual(cacheTimestamp);
     });
   });
 });

Original file line number	Diff line number	Diff line change
`@@ -60,9 +60,9 @@ export const getCachedSheetData =`
`60`	`60`	`TE.map(table =>`
`61`	`61`	`pipe(`
`62`	`62`	`table.rows,`
`63`		`- RA.map(entry => ({`
`64`		`- ...entry,`
`65`		`- cached_data: extractCachedEvents(entry.cached_data),`
	`63`	`+ RA.map(row => ({`
	`64`	`+ ...row,`
	`65`	`+ cached_data: extractCachedEvents(row.cached_data),`
`66`	`66`	`}))`
`67`	`67`	`)`
`68`	`68`	`)`