Merge pull request #1057 from david-roper/large-upload-fix

joshunrau · web-flow · commit e49155ff584c · 2024-12-17T15:24:41.000-05:00
diff --git a/apps/api/src/instrument-records/instrument-records.service.ts b/apps/api/src/instrument-records/instrument-records.service.ts
@@ -11,7 +11,7 @@ import type {
   LinearRegressionResults,
   UploadInstrumentRecordsData
 } from '@opendatacapture/schemas/instrument-records';
-import type { InstrumentRecordModel, Prisma, SessionModel } from '@prisma/generated-client';
+import { type InstrumentRecordModel, Prisma, type SessionModel } from '@prisma/generated-client';
 import { isNumber, pickBy } from 'lodash-es';
 
 import { accessibleQuery } from '@/ability/ability.utils';
@@ -274,91 +274,84 @@ export class InstrumentRecordsService {
       );
     }
 
-    const createdRecordsArray: InstrumentRecordModel[] = [];
     const createdSessionsArray: SessionModel[] = [];
 
     try {
-      for (let i = 0; i < records.length; i++) {
-        const { data: rawData, date, subjectId } = records[i]!;
-        await this.createSubjectIfNotFound(subjectId);
-
-        const session = await this.sessionsService.create({
-          date: date,
-          groupId: groupId ? groupId : null,
-          subjectData: {
-            id: subjectId
-          },
-          type: 'RETROSPECTIVE'
-        });
+      const preProcessedRecords = await Promise.all(
+        records.map(async (record) => {
+          const { data: rawData, date, subjectId } = record;
+
+          // Validate data
+          const parseResult = instrument.validationSchema.safeParse(this.parseJson(rawData));
+          if (!parseResult.success) {
+            console.error(parseResult.error.issues);
+            throw new UnprocessableEntityException(
+              `Data received for record does not pass validation schema of instrument '${instrument.id}'`
+            );
+          }
 
-        createdSessionsArray.push(session);
+          // Ensure subject exists
+          await this.createSubjectIfNotFound(subjectId);
 
-        const sessionId = session.id;
+          const session = await this.sessionsService.create({
+            date: date,
+            groupId: groupId ?? null,
+            subjectData: { id: subjectId },
+            type: 'RETROSPECTIVE'
+          });
 
-        const parseResult = instrument.validationSchema.safeParse(this.parseJson(rawData));
-        if (!parseResult.success) {
-          console.error(parseResult.error.issues);
-          throw new UnprocessableEntityException(
-            `Data received for record at index '${i}' does not pass validation schema of instrument '${instrument.id}'`
-          );
-        }
+          createdSessionsArray.push(session);
 
-        const createdRecord = await this.instrumentRecordModel.create({
-          data: {
-            computedMeasures: instrument.measures
-              ? this.instrumentMeasuresService.computeMeasures(instrument.measures, parseResult.data)
-              : null,
+          const computedMeasures = instrument.measures
+            ? this.instrumentMeasuresService.computeMeasures(instrument.measures, parseResult.data)
+            : null;
+
+          return {
+            computedMeasures,
             data: this.serializeData(parseResult.data),
             date,
-            group: groupId
-              ? {
-                  connect: { id: groupId }
-                }
-              : undefined,
-            instrument: {
-              connect: {
-                id: instrumentId
-              }
-            },
-            session: {
-              connect: {
-                id: sessionId
-              }
-            },
-            subject: {
-              connect: {
-                id: subjectId
-              }
-            }
-          }
-        });
+            groupId,
+            instrumentId,
+            sessionId: session.id,
+            subjectId
+          };
+        })
+      );
 
-        createdRecordsArray.push(createdRecord);
-      }
-    } catch (err) {
-      await this.instrumentRecordModel.deleteMany({
+      await this.instrumentRecordModel.createMany({
+        data: preProcessedRecords
+      });
+
+      return this.instrumentRecordModel.findMany({
         where: {
-          id: {
-            in: createdRecordsArray.map((record) => record.id)
-          }
+          groupId,
+          instrumentId
         }
       });
+    } catch (err) {
       await this.sessionsService.deleteByIds(createdSessionsArray.map((session) => session.id));
       throw err;
     }
-
-    return createdRecordsArray;
   }
 
   private async createSubjectIfNotFound(subjectId: string) {
     try {
-      await this.subjectsService.findById(subjectId);
+      return await this.subjectsService.findById(subjectId);
     } catch (exception) {
       if (exception instanceof NotFoundException) {
         const addedSubject: CreateSubjectDto = {
           id: subjectId
         };
-        await this.subjectsService.create(addedSubject);
+        try {
+          return await this.subjectsService.create(addedSubject);
+        } catch (prismaError) {
+          if (prismaError instanceof Prisma.PrismaClientKnownRequestError && prismaError.code === 'P2002') {
+            console.error(prismaError);
+            return await this.subjectsService.findById(subjectId);
+          } else {
+            throw prismaError;
+          }
+        }
       } else {
         throw exception;
       }
diff --git a/apps/web/src/features/upload/pages/UploadPage.tsx b/apps/web/src/features/upload/pages/UploadPage.tsx
@@ -1,6 +1,6 @@
 import React, { useState } from 'react';
 
-import { Button, FileDropzone, Heading } from '@douglasneuroinformatics/libui/components';
+import { Button, FileDropzone, Heading, Spinner } from '@douglasneuroinformatics/libui/components';
 import { useDownload, useTranslation } from '@douglasneuroinformatics/libui/hooks';
 import { useNotificationsStore } from '@douglasneuroinformatics/libui/hooks';
 import type { AnyUnilingualFormInstrument } from '@opendatacapture/runtime-core';
@@ -16,6 +16,7 @@ import { createUploadTemplateCSV, processInstrumentCSV, reformatInstrumentData }
 
 export const UploadPage = () => {
   const [file, setFile] = useState<File | null>(null);
+  const [isLoading, setIsLoading] = useState(false);
   const download = useDownload();
   const addNotification = useNotificationsStore((store) => store.addNotification);
   const currentGroup = useAppStore((store) => store.currentGroup);
@@ -31,21 +32,44 @@ export const UploadPage = () => {
   };
 
   const handleInstrumentCSV = async () => {
-    const processedDataResult = await processInstrumentCSV(file!, instrument!);
-    if (processedDataResult.success) {
-      const reformattedData = reformatInstrumentData({
-        currentGroup,
-        data: processedDataResult.value,
-        instrument: instrument!
-      });
-      uploadInstrumentRecordsMutation.mutate(reformattedData);
-    } else {
-      addNotification({
-        message: processedDataResult.message,
-        type: 'error'
-      });
+    try {
+      setIsLoading(true);
+      const processedDataResult = await processInstrumentCSV(file!, instrument!);
+      if (processedDataResult.success) {
+        const reformattedData = reformatInstrumentData({
+          currentGroup,
+          data: processedDataResult.value,
+          instrument: instrument!
+        });
+        if (reformattedData.records.length > 1000) {
+          addNotification({
+            message: t({
+              en: 'Lots of entries loading, please wait...',
+              fr: 'Beaucoup de données, veuillez patienter...'
+            }),
+            type: 'info'
+          });
+        }
+        await uploadInstrumentRecordsMutation.mutateAsync(reformattedData);
+      } else {
+        addNotification({
+          message: processedDataResult.message,
+          type: 'error'
+        });
+      }
+      setFile(null);
+    } catch (error) {
+      if (error instanceof Error)
+        addNotification({
+          message: t({
+            en: `An error has happended within the request \n '${error.message}'`,
+            fr: `Une erreur s'est produite lors du téléversement \n '${error.message}'.`
+          }),
+          type: 'error'
+        });
+    } finally {
+      setIsLoading(false);
     }
-    setFile(null);
   };
 
   if (!instrument) {
@@ -62,28 +86,42 @@ export const UploadPage = () => {
           })}
         </Heading>
       </PageHeader>
-      <div className="mx-auto flex w-full max-w-3xl flex-grow flex-col justify-center">
-        <FileDropzone
-          acceptedFileTypes={{
-            'text/csv': ['.csv']
-          }}
-          className="flex h-80 w-full flex-col"
-          file={file}
-          setFile={setFile}
-        />
-        <div className="mt-4 flex justify-between space-x-2">
-          <Button disabled={!(file && instrument)} variant={'primary'} onClick={handleInstrumentCSV}>
-            {t('core.submit')}
-          </Button>
-          <Button className="gap-1.5" disabled={!instrument} variant={'primary'} onClick={handleTemplateDownload}>
-            <DownloadIcon />
-            {t({
-              en: 'Download Template',
-              fr: 'Télécharger le modèle'
-            })}
-          </Button>
+      {!isLoading ? (
+        <div className="mx-auto flex w-full max-w-3xl flex-grow flex-col justify-center">
+          <FileDropzone
+            acceptedFileTypes={{
+              'text/csv': ['.csv']
+            }}
+            className="flex h-80 w-full flex-col"
+            file={file}
+            setFile={setFile}
+          />
+          <div className="mt-4 flex justify-between space-x-2">
+            <Button disabled={!(file && instrument)} variant={'primary'} onClick={handleInstrumentCSV}>
+              {t('core.submit')}
+            </Button>
+            <Button className="gap-1.5" disabled={!instrument} variant={'primary'} onClick={handleTemplateDownload}>
+              <DownloadIcon />
+              {t({
+                en: 'Download Template',
+                fr: 'Télécharger le modèle'
+              })}
+            </Button>
+          </div>
         </div>
-      </div>
+      ) : (
+        <>
+          <div className="mx-auto flex w-full max-w-3xl flex-grow flex-col justify-center">
+            <Spinner className="mx-auto size-1/2"></Spinner>
+            <Heading className="text-center" variant="h3">
+              {t({
+                en: 'Data currently uploading...',
+                fr: 'Données en cours de téléchargement...'
+              })}
+            </Heading>
+          </div>
+        </>
+      )}
     </React.Fragment>
   );
 };
diff --git a/apps/web/src/services/axios.ts b/apps/web/src/services/axios.ts
@@ -13,7 +13,7 @@ axios.interceptors.request.use((config) => {
   config.headers.setAccept('application/json');
 
   // Do not set timeout for setup (can be CPU intensive, especially on slow server)
-  if (config.url !== '/v1/setup') {
+  if (config.url !== '/v1/setup' && config.url !== '/v1/instrument-records/upload') {
     config.timeout = 10000; // abort request after 10 seconds
     config.timeoutErrorMessage = i18n.t({
       en: 'Network Error',
