Persist & serve pnp progress extraction results

Author: CarsonF

src/app.module.ts

@@ -24,6 +24,7 @@ import { PartnershipProducingMediumModule } from './components/partnership-produ
 import { PartnershipModule } from './components/partnership/partnership.module';
 import { PeriodicReportModule } from './components/periodic-report/periodic-report.module';
 import { PinModule } from './components/pin/pin.module';
+import { PnpExtractionResultModule } from './components/pnp/extraction-result/pnp-extraction-result.module';
 import { PostModule } from './components/post/post.module';
 import { ProductProgressModule } from './components/product-progress/product-progress.module';
 import { ProductModule } from './components/product/product.module';
@@ -87,6 +88,7 @@ if (process.env.NODE_ENV !== 'production') {
     PartnershipProducingMediumModule,
     ProgressReportModule,
     PromptsModule,
+    PnpExtractionResultModule,
   ],
 })
 export class AppModule {}

src/components/periodic-report/events/periodic-report-uploaded.event.ts

@@ -1,6 +1,6 @@
 import { LazyGetter as Once } from 'lazy-get-decorator';
 import { Session } from '~/common';
-import { Downloadable, FileNode } from '../../file/dto';
+import { Downloadable, FileVersion } from '../../file/dto';
 import { PnpProgressExtractionResult } from '../../pnp/extraction-result';
 import { PeriodicReport } from '../dto';
 
@@ -10,11 +10,13 @@ import { PeriodicReport } from '../dto';
 export class PeriodicReportUploadedEvent {
   constructor(
     readonly report: PeriodicReport,
-    readonly file: Downloadable<FileNode>,
+    readonly file: Downloadable<FileVersion>,
     readonly session: Session,
   ) {}
 
+  pnpResultUsed = false;
   @Once() get pnpResult() {
+    this.pnpResultUsed = true;
     return new PnpProgressExtractionResult(this.file.id);
   }
 }

src/components/pnp/extraction-result/pnp-extraction-result-progress-report-connection.resolver.ts

@@ -0,0 +1,24 @@
+import { Parent, ResolveField, Resolver } from '@nestjs/graphql';
+import { Loader, LoaderOf } from '@seedcompany/data-loader';
+import { ProgressReport } from '../../progress-report/dto';
+import { PnpProgressExtractionResult } from './extraction-result.dto';
+import { PnpExtractionResultLoader } from './pnp-extraction-result.loader';
+
+@Resolver(ProgressReport)
+export class PnpExtractionResultProgressReportConnectionResolver {
+  @ResolveField(() => PnpProgressExtractionResult, {
+    nullable: true,
+  })
+  async pnpExtractionResult(
+    @Parent() report: ProgressReport,
+    @Loader(() => PnpExtractionResultLoader)
+    loader: LoaderOf<PnpExtractionResultLoader>,
+  ): Promise<PnpProgressExtractionResult | null> {
+    const fileId = report.reportFile.value;
+    if (!fileId) {
+      return null;
+    }
+    const { result } = await loader.load(fileId);
+    return result;
+  }
+}

src/components/pnp/extraction-result/pnp-extraction-result.edgedb.repository.ts

@@ -0,0 +1,21 @@
+import { Injectable } from '@nestjs/common';
+import { ID, NotImplementedException } from '~/common';
+import { CommonRepository } from '~/core/edgedb';
+import { PnpExtractionResult } from './extraction-result.dto';
+import { PnpExtractionResultLoadResult } from './pnp-extraction-result.loader';
+
+@Injectable()
+export class PnpExtractionResultRepository extends CommonRepository {
+  async read(
+    files: ReadonlyArray<ID<'File'>>,
+  ): Promise<readonly PnpExtractionResultLoadResult[]> {
+    throw new NotImplementedException().with(files);
+  }
+
+  async save(
+    file: ID<'FileVersion'>,
+    result: PnpExtractionResult,
+  ): Promise<void> {
+    throw new NotImplementedException().with(file, result);
+  }
+}

src/components/pnp/extraction-result/pnp-extraction-result.loader.ts

@@ -0,0 +1,21 @@
+import { DataLoaderStrategy } from '@seedcompany/data-loader';
+import { ID } from '~/common';
+import { LoaderFactory } from '~/core';
+import { PnpExtractionResult } from './extraction-result.dto';
+import { PnpExtractionResultRepository } from './pnp-extraction-result.edgedb.repository';
+
+export interface PnpExtractionResultLoadResult {
+  id: ID<'File'>;
+  result: PnpExtractionResult | null;
+}
+
+@LoaderFactory()
+export class PnpExtractionResultLoader
+  implements DataLoaderStrategy<PnpExtractionResultLoadResult, ID<'File'>>
+{
+  constructor(private readonly repo: PnpExtractionResultRepository) {}
+
+  async loadMany(files: ReadonlyArray<ID<'File'>>) {
+    return await this.repo.read(files);
+  }
+}

src/components/pnp/extraction-result/pnp-extraction-result.module.ts

@@ -0,0 +1,20 @@
+import { Module } from '@nestjs/common';
+import { splitDb2 } from '~/core';
+import { PnpExtractionResultProgressReportConnectionResolver } from './pnp-extraction-result-progress-report-connection.resolver';
+import { PnpExtractionResultRepository } from './pnp-extraction-result.edgedb.repository';
+import { PnpExtractionResultLoader } from './pnp-extraction-result.loader';
+import { PnpExtractionResultNeo4jRepository } from './pnp-extraction-result.neo4j.repository';
+import { SaveProgressExtractionResultHandler } from './save-progress-extraction-result.handler';
+
+@Module({
+  providers: [
+    PnpExtractionResultProgressReportConnectionResolver,
+    PnpExtractionResultLoader,
+    SaveProgressExtractionResultHandler,
+    splitDb2(PnpExtractionResultRepository, {
+      edge: PnpExtractionResultRepository,
+      neo4j: PnpExtractionResultNeo4jRepository,
+    }),
+  ],
+})
+export class PnpExtractionResultModule {}

src/components/pnp/extraction-result/pnp-extraction-result.neo4j.repository.ts

@@ -0,0 +1,73 @@
+import { Injectable } from '@nestjs/common';
+import { groupToMapBy, mapKeys } from '@seedcompany/common';
+import { inArray, node, relation } from 'cypher-query-builder';
+import { SetNonNullable } from 'type-fest';
+import { ID, PublicOf } from '~/common';
+import { CommonRepository } from '~/core/database';
+import { apoc, merge } from '~/core/database/query';
+import {
+  PnpExtractionResult,
+  PnpProblemSeverity as Severity,
+} from './extraction-result.dto';
+import { PnpExtractionResultRepository } from './pnp-extraction-result.edgedb.repository';
+import { PnpExtractionResultLoadResult } from './pnp-extraction-result.loader';
+
+@Injectable()
+export class PnpExtractionResultNeo4jRepository
+  extends CommonRepository
+  implements PublicOf<PnpExtractionResultRepository>
+{
+  async read(files: ReadonlyArray<ID<'File'>>) {
+    const found = await this.db
+      .query()
+      .match([
+        node('file', 'File'),
+        relation('out', '', 'pnpExtractionResult'),
+        node('result', 'PnpExtractionResult'),
+      ])
+      .where({ 'file.id': inArray(files) })
+      .return<SetNonNullable<PnpExtractionResultLoadResult>>([
+        'file.id as id',
+        merge('result', {
+          problems: apoc.convert.fromJsonList('result.problems'),
+        }).as('result'),
+      ])
+      .run();
+    const map = mapKeys.fromList(found, (r) => r.id).asMap;
+    return files.map((id) => ({
+      id,
+      result: map.get(id)?.result ?? null,
+    }));
+  }
+
+  async save(file: ID<'FileVersion'>, result: PnpExtractionResult) {
+    const bySev = groupToMapBy(result.problems, (p) => p.severity);
+    const stats = Object.fromEntries(
+      [...Severity].flatMap((severity) => [
+        [`result.has${severity}`, bySev.has(severity)],
+        [`result.count${severity}`, bySev.get(severity)?.length ?? 0],
+      ]),
+    );
+
+    await this.db
+      .query()
+      .match([
+        node('file', 'File'),
+        relation('in', 'parent'),
+        node('', 'FileVersion', { id: file }),
+      ])
+      .merge([
+        node('file'),
+        relation('out', '', 'pnpExtractionResult'),
+        node('result', 'PnpExtractionResult'),
+      ])
+      .apply((q) => {
+        q.params.addParam(result.problems, 'problems');
+      })
+      .setValues(stats)
+      .setVariables({
+        'result.problems': 'apoc.convert.toJson($problems)',
+      })
+      .executeAndLogStats();
+  }
+}

src/components/pnp/extraction-result/save-progress-extraction-result.handler.ts

@@ -0,0 +1,16 @@
+import { EventsHandler } from '~/core';
+import { PeriodicReportUploadedEvent } from '../../periodic-report/events';
+import { PnpExtractionResultRepository } from './pnp-extraction-result.edgedb.repository';
+
+@EventsHandler([PeriodicReportUploadedEvent, -1])
+export class SaveProgressExtractionResultHandler {
+  constructor(private readonly repo: PnpExtractionResultRepository) {}
+
+  async handle(event: PeriodicReportUploadedEvent) {
+    if (!event.pnpResultUsed) {
+      return;
+    }
+
+    await this.repo.save(event.file.id, event.pnpResult);
+  }
+}

src/core/database/query/cypher-functions.ts

@@ -76,6 +76,7 @@ export const apoc = {
   convert: {
     /** Converts Neo4j node to object/map of the node's properties */
     toMap: fn1('apoc.convert.toMap'),
+    fromJsonList: fn1('apoc.convert.fromJsonList'),
   },
   create: {
     setLabels: (node: ExpressionInput, labels: readonly string[]) =>