Start scraping quiz events

Author: Lan2u

src/dependencies.ts

@@ -1,8 +1,6 @@
 import {Logger} from 'pino';
 import {Failure, Email, DomainEvent, ResourceVersion} from './types';
 import * as TE from 'fp-ts/TaskEither';
-import * as t from 'io-ts';
-import * as O from 'fp-ts/Option';
 import {FailureWithStatus} from './types/failure-with-status';
 import {StatusCodes} from 'http-status-codes';
 
@@ -40,40 +38,4 @@ export type Dependencies = {
   logger: Logger;
   rateLimitSendingOfEmails: (email: Email) => TE.TaskEither<Failure, Email>;
   sendEmail: (email: Email) => TE.TaskEither<Failure, string>;
-  getCachedSheetData: (sheetId: string) => TE.TaskEither<
-    FailureWithStatus,
-    O.Option<{
-      cached_at: Date;
-      cached_data: t.Validation<
-        ReadonlyArray<
-          | EventOfType<'EquipmentTrainingQuizResult'>
-          | EventOfType<'EquipmentTrainingQuizSync'>
-        >
-      >;
-    }>
-  >;
-  cacheSheetData: (
-    cacheTimestamp: Date,
-    sheetId: GoogleSheetId,
-    logger: Logger,
-    data: ReadonlyArray<
-      | EventOfType<'EquipmentTrainingQuizSync'>
-      | EventOfType<'EquipmentTrainingQuizResult'>
-    >
-  ) => Promise<void>;
-  getCachedTroubleTicketData: (sheetId: string) => TE.TaskEither<
-    FailureWithStatus,
-    O.Option<{
-      cached_at: Date;
-      cached_data: t.Validation<
-        ReadonlyArray<EventOfType<'TroubleTicketResponseSubmitted'>>
-      >;
-    }>
-  >;
-  cacheTroubleTicketData: (
-    cacheTimestamp: Date,
-    sheetId: GoogleSheetId,
-    logger: Logger,
-    data: ReadonlyArray<EventOfType<'TroubleTicketResponseSubmitted'>>
-  ) => Promise<void>;
 };

src/index.ts

@@ -19,12 +19,21 @@ import {initDependencies} from './init-dependencies';
 import * as libsqlClient from '@libsql/client';
 import cookieSession from 'cookie-session';
 import {initRoutes} from './routes';
-import {ensureCachedSheetDataTableExists} from './init-dependencies/google/ensure-cached-sheet-data-table-exists';
 import {
   loadCachedSheetData,
   loadCachedTroubleTicketData,
 } from './load-cached-sheet-data';
 import {timeAsync} from './util';
+import {Worker} from 'worker_threads';
+
+// This background worker can be treated as a completely independent process
+// but we just spawn it as a thread for now.
+const backgroundSyncWorker = new Worker('./training-sheets/sync_worker.js');
+backgroundSyncWorker.on('exit', () => {
+  console.error('Background worker has finished. Stopping parent');
+  // eslint-disable-next-line n/no-process-exit
+  process.exit(1);
+});
 
 // Dependencies and Config
 const conf = loadConfig();
@@ -33,12 +42,7 @@ const dbClient = libsqlClient.createClient({
   syncUrl: conf.TURSO_SYNC_URL,
   authToken: conf.TURSO_TOKEN,
 });
-const cacheClient = libsqlClient.createClient({
-  url: conf.EVENT_DB_URL,
-  syncUrl: conf.TURSO_SYNC_URL,
-  authToken: conf.TURSO_TOKEN,
-});
-const deps = initDependencies(dbClient, cacheClient, conf);
+const deps = initDependencies(dbClient, conf);
 const routes = initRoutes(deps, conf);
 
 // Passport Setup
@@ -106,7 +110,6 @@ server.on('close', () => {
 void (async () => {
   await pipe(
     ensureEventTableExists(dbClient),
-    TE.map(ensureCachedSheetDataTableExists(dbClient)),
     TE.mapLeft(e => deps.logger.error(e, 'Failed to start server'))
   )();
 

src/init-dependencies/google/cache-sheet-data.ts

@@ -0,0 +1,56 @@
+import {Client} from '@libsql/client/.';
+import {dbExecute} from '../../util';
+
+// This table contains a copy of all the training sheet data currently in google.
+// It is read only on requests from the frontend so it can be accelerated via read-replicas.
+// We used to try and turn the google sheet data into events however this proved messier and messier
+// once we needed to do things like cache the sheet data (parsing data is slow and prevents startup before healthcheck failure),
+// do incremental pulls (parsing data is slow), only pull 1 bit of equipment at a time (otherwise it blocks the event loop).
+
+export const ensureSheetDataTableExists = (dbClient: Client) =>
+  dbExecute(
+    dbClient,
+    `
+        CREATE TABLE IF NOT EXISTS sheet_data (
+          sheet_id TEXT PRIMARY KEY,
+          row_index INTEGER,
+          member_number_provided INTEGER,
+          email_provided TEXT,
+          score INTEGER,
+          max_score INTEGER,
+          percentage INTEGER,
+          cached_at INTEGER
+        );
+        `,
+    {}
+  );
+
+export const ensureSheetDataSyncMetadataTableExists = (dbClient: Client) =>
+  dbExecute(
+    dbClient,
+    `
+        CREATE TABLE IF NOT EXISTS sheet_sync_metadata (
+          sheet_id TEXT PRIMARY KEY,
+          last_sync INTEGER
+        );
+        `,
+    {}
+  );
+
+export const ensureTroubleTicketDataTableExists = (dbClient: Client) =>
+  dbExecute(
+    dbClient,
+    `
+        CREATE TABLE IF NOT EXISTS trouble_ticket_data (
+          sheet_id TEXT PRIMARY KEY,
+          response_submitted INTEGER,
+          cached_at INTEGER,
+          submitted_email TEXT,
+          submitted_equipment TEXT,
+          submitted_name TEXT,
+          submitted_membership_number INTEGER,
+          submitted_response_json TEXT
+        );
+        `,
+    {}
+  );

src/init-dependencies/google/cached-data-table.ts

@@ -0,0 +1,51 @@
+import * as t from 'io-ts';
+import * as tt from 'io-ts-types';
+
+export const SheetDataTable = t.strict({
+  rows: t.readonlyArray(
+    t.strict({
+      sheet_id: t.string,
+      row_index: t.Integer,
+      member_number_provided: t.Integer,
+      email_provided: t.string,
+      score: t.Integer,
+      max_score: t.Integer,
+      percentage: t.Integer,
+      cached_at: tt.DateFromNumber,
+    })
+  ),
+});
+export type SheetDataTable = t.TypeOf<typeof SheetDataTable>;
+
+export const SheetSyncMetadataTable = t.strict({
+  rows: t.readonlyArray(
+    t.strict({
+      sheet_id: t.string,
+      row_index: t.Integer,
+      member_number_provided: t.Integer,
+      email_provided: t.string,
+      score: t.Integer,
+      max_score: t.Integer,
+      percentage: t.Integer,
+      cached_at: tt.DateFromNumber,
+    })
+  ),
+});
+export type SheetSyncMetadataTable = t.TypeOf<typeof SheetSyncMetadataTable>;
+
+export const TroubleTicketDataTable = t.strict({
+  rows: t.readonlyArray(
+    t.strict({
+      sheet_id: t.string,
+      response_submitted: tt.DateFromNumber,
+      cached_at: tt.DateFromNumber,
+      // Do not trust provided data - it is not verified.
+      submitted_email: t.union([t.string, t.null]),
+      submitted_equipment: t.union([t.string, t.null]),
+      submitted_name: t.union([t.string, t.null]),
+      submitted_membership_number: t.union([t.Integer, t.null]),
+      submitted_response_json: t.string,
+    })
+  ),
+});
+export type TroubleTicketDataTable = t.TypeOf<typeof TroubleTicketDataTable>;

src/init-dependencies/google/ensure-cached-sheet-data-table-exists.ts

@@ -10,21 +10,10 @@ import {commitEvent} from './event-store/commit-event';
 import {getAllEvents, getAllEventsByType} from './event-store/get-all-events';
 import {getResourceEvents} from './event-store/get-resource-events';
 import {Client} from '@libsql/client';
-import {
-  GoogleHelpers,
-  pullGoogleSheetData,
-  pullGoogleSheetDataMetadata,
-} from './google/pull_sheet_data';
+
 import {initSharedReadModel} from '../read-models/shared-state';
-import {GoogleAuth} from 'google-auth-library';
-import {getCachedSheetData} from './google/get-cached-sheet-data';
-import {cacheSheetData} from './google/cache-sheet-data';
 
-export const initDependencies = (
-  dbClient: Client,
-  cacheClient: Client,
-  conf: Config
-): Dependencies => {
+export const initLogger = (conf: Config) => {
   let loggerOptions: LoggerOptions;
   loggerOptions = {
     formatters: {
@@ -49,8 +38,14 @@ export const initDependencies = (
       },
     };
   }
+  return createLogger(loggerOptions);
+};
 
-  const logger = createLogger(loggerOptions);
+export const initDependencies = (
+  dbClient: Client,
+  conf: Config
+): Dependencies => {
+  const logger = initLogger(conf);
 
   const emailTransporter = nodemailer.createTransport(
     smtp({
@@ -64,35 +59,10 @@ export const initDependencies = (
     })
   );
 
-  let googleHelpers: O.Option<GoogleHelpers> = O.none;
-  if (
-    conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON.toLowerCase().trim() !== 'disabled'
-  ) {
-    const googleAuth = new GoogleAuth({
-      // Google issues the credentials file and validates it.
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-      credentials: JSON.parse(conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON),
-      scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'],
-    });
-    googleHelpers = O.some({
-      pullGoogleSheetData: pullGoogleSheetData(googleAuth),
-      pullGoogleSheetDataMetadata: pullGoogleSheetDataMetadata(googleAuth),
-    });
-  }
-
-  const _cacheSheetData: Dependencies['cacheSheetData'] =
-    cacheSheetData(cacheClient);
-  const _cacheTroubleTicketData: Dependencies['cacheTroubleTicketData'] =
-    cacheSheetData(cacheClient);
-
   const sharedReadModel = initSharedReadModel(
     dbClient,
     logger,
-    googleHelpers,
-    conf.GOOGLE_RATELIMIT_MS,
     O.fromNullable(conf.TROUBLE_TICKET_SHEET),
-    _cacheSheetData,
-    _cacheTroubleTicketData,
     O.fromNullable(conf.RECURLY_TOKEN)
   );
 
@@ -105,10 +75,6 @@ export const initDependencies = (
     rateLimitSendingOfEmails: createRateLimiter(5, 24 * 3600),
     sendEmail: sendEmail(emailTransporter, conf.SMTP_FROM),
     logger,
-    getCachedSheetData: getCachedSheetData(dbClient),
-    getCachedTroubleTicketData: getCachedSheetData(dbClient),
-    cacheSheetData: _cacheSheetData,
-    cacheTroubleTicketData: _cacheTroubleTicketData,
   };
   return deps;
 };