Fix cache wipe issue demontrated in previous commit integration test

Author: Lan2u

src/read-models/shared-state/async-apply-external-event-sources.ts

@@ -3,7 +3,6 @@ import * as E from 'fp-ts/Either';
 import * as O from 'fp-ts/Option';
 import {DomainEvent} from '../../types';
 import {BetterSQLite3Database} from 'drizzle-orm/better-sqlite3';
-import {EpochTimestampMilliseconds} from './return-types';
 
 import {constructEvent, EventOfType} from '../../types/domain-event';
 import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data';
@@ -22,7 +21,6 @@ import {
 } from '../../training-sheets/google';
 import {getChunkIndexes} from '../../util';
 import {UUID} from 'io-ts-types';
-import {expandLastQuizResult} from './equipment/expand';
 
 const ROW_BATCH_SIZE = 200;
 
@@ -31,7 +29,6 @@ const pullNewEquipmentQuizResultsForSheet = async (
   googleHelpers: GoogleHelpers,
   equipmentId: UUID,
   trainingSheetId: string,
-  eventsFromExclusive: O.Option<EpochTimestampMilliseconds>,
   sheet: GoogleSheetMetadata,
   timezone: string,
   updateState: (event: EventOfType<'EquipmentTrainingQuizResult'>) => void
@@ -71,8 +68,7 @@ const pullNewEquipmentQuizResultsForSheet = async (
       trainingSheetId,
       equipmentId,
       sheet,
-      timezone,
-      eventsFromExclusive
+      timezone
     )(data.right);
     logger.info(
       'Google sheet data extracted, updating data with the extracted data...'
@@ -89,7 +85,6 @@ export const pullNewEquipmentQuizResults = async (
   googleHelpers: GoogleHelpers,
   equipmentId: UUID,
   trainingSheetId: string,
-  eventsSinceExclusive: O.Option<EpochTimestampMilliseconds>,
   updateState: (
     event:
       | EventOfType<'EquipmentTrainingQuizSync'>
@@ -155,7 +150,6 @@ export const pullNewEquipmentQuizResults = async (
       googleHelpers,
       equipmentId,
       trainingSheetId,
-      eventsSinceExclusive,
       sheet,
       initialMeta.right.properties.timeZone,
       updateState
@@ -220,7 +214,6 @@ export const asyncApplyExternalEventSources = (
         googleHelpers.value,
         equipment.id,
         equipment.trainingSheetId.value,
-        expandLastQuizResult(currentState)(equipment).lastQuizResult,
         collectEvents
       );
       equipmentLogger.info(

src/read-models/shared-state/equipment/expand.ts

@@ -228,7 +228,7 @@ const expandOrphanedTrainingQuizes =
     ),
   });
 
-export const expandLastQuizResult =
+const expandLastQuizResult =
   (db: BetterSQLite3Database) =>
   <T extends MinimalEquipment>(
     equipment: T

src/training-sheets/google.ts

@@ -262,10 +262,7 @@ export const extractGoogleSheetData =
     trainingSheetId: string,
     equipmentId: UUID,
     metadata: GoogleSheetMetadata,
-    timezone: string,
-    // Note we filter events on timestamp rather than last row currently to handle
-    // blank rows but potentially we could switch if we added detection for blank rows.
-    eventsFromExclusive: O.Option<EpochTimestampMilliseconds>
+    timezone: string
   ) =>
   (
     spreadsheet: GoogleSpreadsheetDataForSheet
@@ -287,12 +284,7 @@ export const extractGoogleSheetData =
                 timezone
               )
             ),
-            RA.filterMap(e => e),
-            RA.filter(
-              e =>
-                O.isNone(eventsFromExclusive) ||
-                e.timestampEpochMS > eventsFromExclusive.value
-            )
+            RA.filterMap(e => e)
           )
         )
       )

tests/training-sheets/extract-google-sheet-data.test.ts

@@ -21,8 +21,7 @@ describe('extractGoogleSheetData', () => {
           memberNumber: O.some(2),
         },
       },
-      'Europe/London',
-      O.none
+      'Europe/London'
     )({
       sheets: [
         {

tests/training-sheets/integration.test.ts

@@ -12,7 +12,6 @@ import {
 } from '../../src/init-dependencies/google/pull_sheet_data';
 import {GoogleAuth} from 'google-auth-library/build/src/auth/googleauth';
 import {UUID} from 'io-ts-types';
-import * as O from 'fp-ts/Option';
 import * as RA from 'fp-ts/ReadonlyArray';
 import * as N from 'fp-ts/number';
 import {DomainEvent} from '../../src/types';
@@ -44,7 +43,6 @@ const getEvents = async (trainingSheetId: string) => {
     },
     TEST_EQUIPMENT_ID,
     trainingSheetId,
-    O.none,
     event => events.push(event)
   );
   return events;

tests/training-sheets/process-events.test.ts

@@ -7,10 +7,8 @@ import {
 import pino from 'pino';
 import * as RA from 'fp-ts/lib/ReadonlyArray';
 import * as N from 'fp-ts/number';
-import * as O from 'fp-ts/Option';
 import * as gsheetData from '../data/google_sheet_data';
 import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources';
-import {EpochTimestampMilliseconds} from '../../src/read-models/shared-state/return-types';
 import {localGoogleHelpers} from '../init-dependencies/pull-local-google';
 
 const sortQuizResults = RA.sort({
@@ -28,8 +26,7 @@ const sortQuizResults = RA.sort({
 
 const pullNewEquipmentQuizResultsLocal = async (
   equipmentId: UUID,
-  trainingSheetId: string,
-  eventsSinceExclusive: O.Option<EpochTimestampMilliseconds>
+  trainingSheetId: string
 ) => {
   const newEvents: DomainEvent[] = [];
   await pullNewEquipmentQuizResults(
@@ -40,7 +37,6 @@ const pullNewEquipmentQuizResultsLocal = async (
     localGoogleHelpers,
     equipmentId,
     trainingSheetId,
-    eventsSinceExclusive,
     newEvent => {
       newEvents.push(newEvent);
     }
@@ -57,14 +53,12 @@ type EquipmentQuizResultEvents = {
   endTime: Date;
 };
 const pullEquipmentQuizResultsWrapper = async (
-  spreadsheetId: string,
-  lastQuizResult: O.Option<EpochTimestampMilliseconds> = O.none
+  spreadsheetId: string
 ): Promise<EquipmentQuizResultEvents> => {
   const startTime = new Date();
   const events = await pullNewEquipmentQuizResultsLocal(
     TEST_EQUIPMENT_ID,
-    spreadsheetId,
-    lastQuizResult
+    spreadsheetId
   );
   const endTime = new Date();
   const result = {
@@ -175,66 +169,6 @@ describe('Training sheets worker', () => {
           >(expectedEvent);
         }
       });
-      it('Only take new rows, date in future', async () => {
-        const results = await pullEquipmentQuizResultsWrapper(
-          gsheetData.BAMBU.apiResp.spreadsheetId!,
-          O.some(Date.now() as EpochTimestampMilliseconds)
-        );
-        checkQuizSync(results);
-        expect(results.quizResults).toHaveLength(0);
-      });
-      it('Only take new rows, date in far past', async () => {
-        const results = await pullEquipmentQuizResultsWrapper(
-          gsheetData.BAMBU.apiResp.spreadsheetId!,
-          O.some(0 as EpochTimestampMilliseconds)
-        );
-        checkQuizSync(results);
-        expect(results.quizResults).toHaveLength(
-          gsheetData.BAMBU.entries.length
-        );
-      });
-
-      // The quiz results have dates:
-      // 1700768963 Thursday, November 23, 2023 7:49:23 PM
-      // 1700769348 Thursday, November 23, 2023 7:55:48 PM
-      // 1710249052 Tuesday, March 12, 2024 1:10:52 PM
-      // 1710249842 Tuesday, March 12, 2024 1:24:02 PM
-
-      it('Only take new rows, exclude 1', async () => {
-        const results = await pullEquipmentQuizResultsWrapper(
-          gsheetData.BAMBU.apiResp.spreadsheetId!,
-          O.some(1700768963_000 as EpochTimestampMilliseconds)
-        );
-        checkQuizSync(results);
-        expect(results.quizResults).toHaveLength(3);
-      });
-
-      it('Only take new rows, exclude 2', async () => {
-        const results = await pullEquipmentQuizResultsWrapper(
-          gsheetData.BAMBU.apiResp.spreadsheetId!,
-          O.some(1700769348_000 as EpochTimestampMilliseconds)
-        );
-        checkQuizSync(results);
-        expect(results.quizResults).toHaveLength(2);
-      });
-
-      it('Only take new rows, exclude 3', async () => {
-        const results = await pullEquipmentQuizResultsWrapper(
-          gsheetData.BAMBU.apiResp.spreadsheetId!,
-          O.some(1710249052_000 as EpochTimestampMilliseconds)
-        );
-        checkQuizSync(results);
-        expect(results.quizResults).toHaveLength(1);
-      });
-
-      it('Only take new rows, exclude all (already have latest)', async () => {
-        const results = await pullEquipmentQuizResultsWrapper(
-          gsheetData.BAMBU.apiResp.spreadsheetId!,
-          O.some(1710249842_000 as EpochTimestampMilliseconds)
-        );
-        checkQuizSync(results);
-        expect(results.quizResults).toHaveLength(0);
-      });
     });
   });
 });