Try syncing a single piece of equipment at a time

Author: Lan2u

src/read-models/shared-state/async-apply-external-event-sources.ts

@@ -15,7 +15,7 @@ import {
   GoogleSpreadsheetDataForSheet,
 } from '../../init-dependencies/google/pull_sheet_data';
 
-import {getAllEquipmentMinimal} from './equipment/get';
+import {getLeastRecentlySyncedEquipment} from './equipment/get';
 import {Dependencies} from '../../dependencies';
 import {
   extractGoogleSheetMetadata,
@@ -418,14 +418,19 @@ async function asyncApplyGoogleEvents(
         logger.info('...done');
       }
 
-      logger.info('Pulling google training sheet data...');
-      for (const equipment of getAllEquipmentMinimal(currentState)) {
+      logger.info(
+        'Pulling google training sheet data for least recently synced equipment'
+      );
+      // Temporarily try syncing just 1 piece of equipment at a time
+      for (const equipment of getLeastRecentlySyncedEquipment(
+        currentState,
+        1
+      )) {
         const equipmentLogger = logger.child({equipment});
         if (
           O.isNone(equipment.trainingSheetId) ||
           (O.isSome(equipment.lastQuizSync) &&
-            Date.now() - equipment.lastQuizSync.value <
-              googleRefreshIntervalMs + Math.random() * googleRefreshIntervalMs) // Try a random offset to spread out cpu usage.
+            Date.now() - equipment.lastQuizSync.value < googleRefreshIntervalMs)
         ) {
           equipmentLogger.info('No google training sheet refresh required');
           continue;
@@ -478,7 +483,7 @@ async function asyncApplyGoogleEvents(
           }
         );
       }
-      logger.info('...done');
+      logger.info('Finished pulling google training sheet data');
     }
   );
 }

src/read-models/shared-state/equipment/get.ts

@@ -1,6 +1,6 @@
 import {pipe} from 'fp-ts/lib/function';
 import {BetterSQLite3Database} from 'drizzle-orm/better-sqlite3';
-import {eq} from 'drizzle-orm';
+import {eq, asc, isNotNull} from 'drizzle-orm';
 import * as O from 'fp-ts/Option';
 import * as RA from 'fp-ts/ReadonlyArray';
 import {equipmentTable} from '../state';
@@ -51,3 +51,18 @@ export const getAllEquipmentMinimal = (
   db: BetterSQLite3Database
 ): ReadonlyArray<MinimalEquipment> =>
   pipe(db.select().from(equipmentTable).all(), RA.map(transformRow));
+
+export const getLeastRecentlySyncedEquipment = (
+  db: BetterSQLite3Database,
+  count: number
+): ReadonlyArray<MinimalEquipment> =>
+  pipe(
+    db
+      .select()
+      .from(equipmentTable)
+      .where(isNotNull(equipmentTable.trainingSheetId))
+      .orderBy(asc(equipmentTable.lastQuizSync))
+      .limit(count)
+      .all(),
+    RA.map(transformRow)
+  );

tests/google-sheets/async-apply-external.test.ts

@@ -1,4 +1,5 @@
 import * as O from 'fp-ts/Option';
+import * as RA from 'fp-ts/ReadonlyArray';
 import {NonEmptyString, UUID} from 'io-ts-types';
 import {faker} from '@faker-js/faker';
 import * as gsheetData from '../data/google_sheet_data';
@@ -70,22 +71,38 @@ describe('Integration asyncApplyExternalEventSources', () => {
       areaId,
       O.some(gsheetData.METAL_LATHE.apiResp.spreadsheetId!)
     );
-    const results = await runAsyncApplyExternalEventSources(framework);
-    checkLastQuizSyncUpdated(results);
+    const resultsAfterFirstRun =
+      await runAsyncApplyExternalEventSources(framework);
+
+    const resultsAfterSecondRun =
+      await runAsyncApplyExternalEventSources(framework);
+
+    expect(
+      RA.filter((e: Equipment) => O.isSome(e.lastQuizSync))(
+        Array.from(resultsAfterFirstRun.equipmentAfter.values())
+      )
+    ).toHaveLength(1); // Only update a single piece of equipment at a time.
+
+    expect(
+      RA.filter((e: Equipment) => O.isSome(e.lastQuizSync))(
+        Array.from(resultsAfterSecondRun.equipmentAfter.values())
+      )
+    ).toHaveLength(2); // Only update a single piece of equipment at a time.
+
     checkLastQuizEventTimestamp(
       gsheetData.BAMBU,
-      results.equipmentAfter.get(bambu.id)!
+      resultsAfterSecondRun.equipmentAfter.get(bambu.id)!
     );
     checkLastQuizEventTimestamp(
       gsheetData.METAL_LATHE,
-      results.equipmentAfter.get(lathe.id)!
+      resultsAfterSecondRun.equipmentAfter.get(lathe.id)!
     );
 
     // We already test the produced quiz result events above
     // and testing updateState is also tested elsewhere so this integration
     // test doesn't need to enumerate every combination it just needs to check
     // that generally the equipment is getting updated.
-    const bambuAfter = results.equipmentAfter.get(bambu.id)!;
+    const bambuAfter = resultsAfterSecondRun.equipmentAfter.get(bambu.id)!;
     expect(bambuAfter.orphanedPassedQuizes).toHaveLength(0);
     expect(bambuAfter.membersAwaitingTraining).toHaveLength(1);
     expect(bambuAfter.membersAwaitingTraining[0].memberNumber).toStrictEqual(
@@ -96,7 +113,7 @@ describe('Integration asyncApplyExternalEventSources', () => {
     );
 
     // Lathe results only have a single failed entry.
-    const latheAfter = results.equipmentAfter.get(lathe.id)!;
+    const latheAfter = resultsAfterSecondRun.equipmentAfter.get(lathe.id)!;
     expect(latheAfter.orphanedPassedQuizes).toHaveLength(0);
     expect(latheAfter.failedQuizAttempts).toHaveLength(1);
     expect(latheAfter.failedQuizAttempts[0]).toMatchObject({