Fix pulling bambu x1 data issue with missing row data

Author: Lan2u

src/init-dependencies/google/pull_sheet_data.ts

@@ -52,17 +52,21 @@ export const GoogleSpreadsheetDataForSheet = t.strict({
   sheets: tt.nonEmptyArray(
     // Array always has length = 1 because this is data for a single sheet.
     t.strict({
-      data: tt.nonEmptyArray(
+      data: t.array(
         t.strict({
-          rowData: tt.nonEmptyArray(
-            t.strict({
-              values: tt.nonEmptyArray(
-                t.strict({
-                  formattedValue: tt.withFallback(t.string, ''),
-                })
-              ),
-            })
-          ),
+          rowData: t.union([
+            t.array(
+              t.strict({
+                values: t.array(
+                  t.strict({
+                    formattedValue: tt.withFallback(t.string, ''),
+                  })
+                ),
+              })
+            ),
+            t.undefined,
+            t.null,
+          ]),
         })
       ),
     })

src/read-models/shared-state/async-apply-external-event-sources.ts

@@ -63,7 +63,8 @@ const pullNewEquipmentQuizResultsForSheet = async (
       maxCol
     )();
     if (E.isLeft(data)) {
-      logger.debug(
+      logger.error(
+        data.left,
         'Failed to pull data for sheet %s rows %s to %s, skipping rest of sheet'
       );
       return;

src/training-sheets/extract-metadata.ts

@@ -3,6 +3,8 @@ import * as O from 'fp-ts/Option';
 
 import {Logger} from 'pino';
 import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data';
+import {array} from 'fp-ts';
+import {pipe} from 'fp-ts/lib/function';
 
 const EMAIL_COLUMN_NAMES = ['email address', 'email'];
 
@@ -37,22 +39,29 @@ export const extractGoogleSheetMetadata =
     firstRowData: GoogleSpreadsheetDataForSheet
   ): O.Option<GoogleSheetMetadata> => {
     logger = logger.child({sheetName: initialMeta.properties.title});
-    const columnNames = firstRowData.sheets[0].data[0].rowData[0].values.map(
-      col => col.formattedValue
+    const columnNames: O.Option<string[]> = pipe(
+      firstRowData.sheets[0].data,
+      array.lookup(0),
+      O.flatMap(sheetData => O.fromNullable(sheetData.rowData)),
+      O.flatMap(array.lookup(0)),
+      O.flatMap(firstRowData =>
+        O.some(firstRowData.values.map(col => col.formattedValue))
+      )
     );
+    if (O.isNone(columnNames)) {
+      logger.error('Found no column names for sheet, skipping sheet');
+      return O.none;
+    }
     logger.trace('Found column names for sheet: %o', columnNames);
     const timestamp = RA.findIndex<string>(
       val => val.toLowerCase() === 'timestamp'
-    )(columnNames);
+    )(columnNames.value);
     if (O.isNone(timestamp)) {
-      logger.warn(
-        'Failed to find timestamp column, skipping sheet: %s',
-        initialMeta.properties.title
-      );
+      logger.warn('Failed to find timestamp column, skipping sheet');
       return O.none;
     }
     const score = RA.findIndex<string>(val => val.toLowerCase() === 'score')(
-      columnNames
+      columnNames.value
     );
     if (O.isNone(score)) {
       logger.warn(
@@ -63,10 +72,10 @@ export const extractGoogleSheetMetadata =
     }
     const memberNumber = RA.findIndex<string>(
       val => val.toLowerCase() === 'membership number'
-    )(columnNames);
+    )(columnNames.value);
     const email = RA.findIndex<string>(val =>
       EMAIL_COLUMN_NAMES.includes(val.toLowerCase())
-    )(columnNames);
+    )(columnNames.value);
 
     return O.some({
       name: initialMeta.properties.title,

src/training-sheets/google.ts

@@ -12,6 +12,7 @@ import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-typ
 import {GoogleSheetMetadata} from './extract-metadata';
 import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data';
 import {lookup} from 'fp-ts/ReadonlyArray';
+import {array} from 'fp-ts';
 
 // Bounds to prevent clearly broken parsing.
 const MIN_RECOGNISED_MEMBER_NUMBER = 0;
@@ -270,16 +271,35 @@ export const extractGoogleSheetData =
     spreadsheet: GoogleSpreadsheetDataForSheet
   ): ReadonlyArray<EventOfType<'EquipmentTrainingQuizResult'>> => {
     return pipe(
-      spreadsheet.sheets[0].data[0].rowData,
-      RA.map(
-        extractFromRow(logger, metadata, equipmentId, trainingSheetId, timezone)
+      spreadsheet.sheets[0].data,
+      array.lookup(0),
+      O.flatMap(sheetData => O.fromNullable(sheetData.rowData)),
+      O.flatMap(row_data =>
+        O.some(
+          pipe(
+            row_data,
+            RA.map(
+              extractFromRow(
+                logger,
+                metadata,
+                equipmentId,
+                trainingSheetId,
+                timezone
+              )
+            ),
+            RA.filterMap(e => e),
+            RA.filter(
+              e =>
+                O.isNone(eventsFromExclusive) ||
+                e.timestampEpochMS > eventsFromExclusive.value
+            )
+          )
+        )
       ),
-      RA.filterMap(e => e),
-      RA.filter(
-        e =>
-          O.isNone(eventsFromExclusive) ||
-          e.timestampEpochMS > eventsFromExclusive.value
-      )
+      O.getOrElse(() => {
+        logger.warn('Failed to get row data, skipping');
+        return [] as ReadonlyArray<EventOfType<'EquipmentTrainingQuizResult'>>;
+      })
     );
   };
 

tests/init-dependencies/pull-local-google.ts

@@ -33,10 +33,11 @@ const localPullGoogleSheetData = (
     gsheetData.TRAINING_SHEETS[trainingSheetId].sheets[sheetName]
   );
   if (sheet) {
-    if (rowStart > sheet.sheets[0].data[0].rowData.length) {
+    const rowData = sheet.sheets[0].data[0].rowData!;
+    if (rowStart > rowData.length) {
       return TE.left('Sheet no more data');
     }
-    sheet.sheets[0].data[0].rowData = sheet.sheets[0].data[0].rowData.slice(
+    sheet.sheets[0].data[0].rowData = rowData.slice(
       rowStart - 1, // 1 indexed.
       rowEnd
     ) as NonEmptyArray<{

tests/training-sheets/integration.test.ts

@@ -37,7 +37,7 @@ const getEvents = async (trainingSheetId: string) => {
   });
   const events: DomainEvent[] = [];
   await pullNewEquipmentQuizResults(
-    pino({level: 'silent'}),
+    pino({level: 'trace'}),
     {
       pullGoogleSheetData: pullGoogleSheetData(auth),
       pullGoogleSheetDataMetadata: pullGoogleSheetDataMetadata(auth),
@@ -189,6 +189,21 @@ describe('Google training sheet integration', () => {
 
     expect(userEvents).toStrictEqual(expected);
   });
+  it('Bambu X1', async () => {
+    const events = await getEvents(
+      '1i1vJmCO8_Dkpbv-izOSkffoAeJTNrJsmAV5hD0w2ADw'
+    );
+    expect(events).toHaveLength(729); // Obviously this will change so you will need to manually update.
+    const userEvents = pipe(
+      events,
+      filterUserEvents(TEST_USER),
+      RA.sort(ordByTimestampEpoch),
+      redactEmail,
+      stripNonStatic
+    );
+    console.log(userEvents);
+    expect(userEvents).toHaveLength(3);
+  }, 30000);
 
   // it('Metal Lathe', async () => {
   //   const events = await getEvents(
@@ -284,9 +299,4 @@ describe('Google training sheet integration', () => {
   //     '1yulN3ewYS2XpT22joP5HteZ9H9qebvSEcFXQhxPwXlk'
   //   );
   // });
-  // it.skip('Bambu', async () => {
-  //   const _events = await getEvents(
-  //     '1i1vJmCO8_Dkpbv-izOSkffoAeJTNrJsmAV5hD0w2ADw'
-  //   );
-  // });
 });