networkDesign backend: checkpoint data preparation and recovery

This adds proper data in internal_data to easily recover the services
created for the network design job and the line's level of services from
the capnp cache.

Author: tahini

packages/transition-backend/src/services/networkDesign/transitNetworkDesign/evolutionary/EvolutionaryTransitNetworkDesignJob.ts

@@ -24,9 +24,10 @@ import Generation from '../../../evolutionaryAlgorithm/generation/Generation';
 import * as AlgoTypes from '../../../evolutionaryAlgorithm/internalTypes';
 import LineAndNumberOfVehiclesGeneration, { generateFirstCandidates, reproduceCandidates } from '../../../evolutionaryAlgorithm/generation/LineAndNumberOfVehiclesGeneration';
 import TrError from 'chaire-lib-common/lib/utils/TrError';
-import { collectionToCache as serviceCollectionToCache } from '../../../../models/capnpCache/transitServices.cache.queries';
-import { objectsToCache as linesToCache } from '../../../../models/capnpCache/transitLines.cache.queries';
+import { collectionToCache as serviceCollectionToCache, collectionFromCache as serviceCollectionFromCache } from '../../../../models/capnpCache/transitServices.cache.queries';
+import { objectsToCache as linesToCache, objectFromCache as lineFromCache } from '../../../../models/capnpCache/transitLines.cache.queries';
 import { prepareServices, saveSimulationScenario } from '../../../evolutionaryAlgorithm/preparation/ServicePreparation';
+import Line from 'transition-common/lib/services/line/Line';
 
 /**
  * Do batch calculation on a csv file input
@@ -67,6 +68,36 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
         this.options = this.parameters.algorithmConfiguration.config;
     }
 
+    private _getSimulatedLineCollection = (collections: {
+        lines: LineCollection;
+        agencies: AgencyCollection;
+        services: ServiceCollection;
+    }): LineCollection => {
+        const {
+            nonSimulatedServices,
+            simulatedAgencies,
+            linesToKeep: linesToKeepParam
+        } = this.parameters.transitNetworkDesignParameters;
+        const linesToKeep = linesToKeepParam || [];
+
+        const agencies = simulatedAgencies?.map((agencyId) => collections.agencies.getById(agencyId)) || [];
+        const lines = agencies.flatMap((agency) => (agency ? agency.getLines() : []));
+
+        // Sort lines so lines to keep are at the beginning
+        console.log('Sorting lines...');
+        lines.sort((lineA, lineB) =>
+            linesToKeep.includes(lineA.getId()) && linesToKeep.includes(lineB.getId())
+                ? 0
+                : linesToKeep.includes(lineA.getId())
+                    ? -1
+                    : linesToKeep.includes(lineB.getId())
+                        ? 1
+                        : 0
+        );
+        const simulatedLineCollection = new LineCollection(lines, {});
+        return simulatedLineCollection;
+    }
+
     /**
      * Prepare the data for the current simulation: Create services for the
      * lines to simulate and prepare collections containing only the required
@@ -98,28 +129,8 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
          */
         simulatedServices: ServiceCollection;
     }> => {
-        const {
-            nonSimulatedServices,
-            simulatedAgencies,
-            linesToKeep: linesToKeepParam
-        } = this.parameters.transitNetworkDesignParameters;
-        const linesToKeep = linesToKeepParam || [];
-
-        const agencies = simulatedAgencies?.map((agencyId) => collections.agencies.getById(agencyId)) || [];
-        const lines = agencies.flatMap((agency) => (agency ? agency.getLines() : []));
-
-        // Sort lines so lines to keep are at the beginning
-        console.log('Sorting lines...');
-        lines.sort((lineA, lineB) =>
-            linesToKeep.includes(lineA.getId()) && linesToKeep.includes(lineB.getId())
-                ? 0
-                : linesToKeep.includes(lineA.getId())
-                    ? -1
-                    : linesToKeep.includes(lineB.getId())
-                        ? 1
-                        : 0
-        );
-        const simulatedLineCollection = new LineCollection(lines, {});
+    
+        const simulatedLineCollection = this._getSimulatedLineCollection(collections);
 
         // Prepare various services for lines
         console.log('Preparing services...');
@@ -146,7 +157,7 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
 
         // Add non simulated services to the collection, ie those used by the simulation, but not generated by it
         const existingServices =
-            nonSimulatedServices
+            this.parameters.transitNetworkDesignParameters.nonSimulatedServices
                 ?.map((serviceId) => collections.services.getById(serviceId))
                 .filter((service) => service !== undefined) || [];
         existingServices.forEach((service) => services.add(service as Service));
@@ -162,55 +173,103 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
         return this.currentIteration > (this.options.numberOfGenerations || 0);
     };
 
-    private _run = async (
-
-    ): Promise<boolean> => {
+    private _loadAndPrepareData = async() :Promise<void> => {
 
-         // Load the necessary data from the server
-         const jobId = this.job.id;
-         console.time(`Preparing data for evolutionary transit network design job ${jobId}`);
-         // FIXME This loads everything! we don't need all that
-         await this.loadServerData(serviceLocator.socketEventManager);
-         console.timeEnd(`Preparing data for evolutionary transit network design job ${jobId}`);
+        // Load the necessary data from the server
+        const jobId = this.job.id;
+        console.time(`Preparing data for evolutionary transit network design job ${jobId}`);
+        // FIXME This loads everything! we don't need all that
+        await this.loadServerData(serviceLocator.socketEventManager);
+        console.timeEnd(`Preparing data for evolutionary transit network design job ${jobId}`);
 
-         // Prepare the cache data for this job
-         // FIXME Add checkpoint here
-         console.time(`Preparing cache directory for job ${jobId}`);
-         this.prepareCacheDirectory();
-         console.timeEnd(`Preparing cache directory for job ${jobId}`);
-
-        
-         console.time(`Running evolutionary transit network design algorithm for job ${jobId}`);
-         console.timeEnd(`Running evolutionary transit network design algorithm for job ${jobId}`);
+        // Prepare the cache data for this job
+        // FIXME Add checkpoint here
+        console.time(`Preparing cache directory for job ${jobId}`);
+        this.prepareCacheDirectory();
+        console.timeEnd(`Preparing cache directory for job ${jobId}`);
 
         // FIXME Use a seed from the job data?
         const randomGenerator = random;
         // Get the agencies data
 
-        // FIXME Cache preparation and service preparation are linked. There should be a checkpoint after and data reloaded from there
-        // FIXME2 see if we need to keep anything in checkpoints
-        // FIXME3 Some of them might be in the class instead
         const { simulatedLineCollection, lineServices } =
             await this.prepareData({ agencies: this.agencyCollection, lines: this.allLineCollection, services: this.serviceCollection });
         this.setLineServices(lineServices);
         this.simulatedLineCollection = simulatedLineCollection;
-        // FIXME There's something to checkpoint here
 
         // Initialize population size if not done yet, as well as results
-        if (this.job.attributes.internal_data.populationSize === undefined) {
-            const populationSize = randomInRange(
-                [this.options.populationSizeMin, this.options.populationSizeMax],
-                randomGenerator
-            );
-            this.job.attributes.internal_data.populationSize = populationSize;
-            // FIXME Make sure results structure is well defined
-            const algorithmResults: { generations: ResultSerialization[], scenarioIds: string[] } = { generations: [], scenarioIds: [] };
-            this.job.attributes.data.results = algorithmResults;
-            this.job.save(this.executorOptions.progressEmitter);
+        const populationSize = randomInRange(
+            [this.options.populationSizeMin, this.options.populationSizeMax],
+            randomGenerator
+        );
+
+        // Checkoint the data preparation and initialize results
+        this.job.attributes.internal_data.populationSize = populationSize;
+        this.job.attributes.internal_data.dataPrepared = true;
+        this.job.attributes.internal_data.lineServices = Object.keys(lineServices).reduce((acc, lineId) => {
+            acc[lineId] = lineServices[lineId].map((lvlOfService) => ({
+                serviceId: lvlOfService.service.getId(),
+                numberOfVehicles: lvlOfService.numberOfVehicles
+            }));
+            return acc;
+        }, {});
+
+        // FIXME Make sure results structure is well defined
+        const algorithmResults: { generations: ResultSerialization[], scenarioIds: string[] } = { generations: [], scenarioIds: [] };
+        this.job.attributes.data.results = algorithmResults;
+        this.job.save(this.executorOptions.progressEmitter);
+    }
+
+    private _loadAndPrepareDataFromCache = async() :Promise<void> => {
+        // Load the necessary data from the server
+        const jobId = this.job.id;
+        console.time(`Preparing data for evolutionary transit network design job from cache ${jobId}`);
+        // FIXME Do we even need this? Or can we just start recovery at serviceCollectionFromCache call and get only required data?
+        await this.loadServerData(serviceLocator.socketEventManager);
+        
+        // Get the simulated lines
+        const simulatedLineCollection = this._getSimulatedLineCollection({ agencies: this.agencyCollection, lines: this.allLineCollection, services: this.serviceCollection });
+        this.simulatedLineCollection = simulatedLineCollection;
+
+        // Read the services from the cache, with all individual line services
+        // Save services and lines, with their schedules, to cache
+        const serviceCollection = await serviceCollectionFromCache(this.getCacheDirectory());
+        this.serviceCollection.setFeatures(serviceCollection.getFeatures());
+        // Read each of the simulation line from cache
+        for (let i = 0; i < simulatedLineCollection.getFeatures().length; i++) {
+            const line = simulatedLineCollection.getFeatures()[i];
+            const lineWithSchedule = await lineFromCache(line.getId(), this.getCacheDirectory()) as Line;
+            simulatedLineCollection.updateFeature(lineWithSchedule);
+        }
+        // Recreate the line level of services
+        const lineServicesSerialized = this.job.attributes.internal_data.lineServices || {};
+        const lineServices: AlgoTypes.LineServices = Object.keys(lineServicesSerialized).reduce((acc, lineId) => {
+            acc[lineId] = lineServicesSerialized[lineId].map((lvlOfService) => {
+                const service = serviceCollection.getById(lvlOfService.serviceId);
+                if (!service) {
+                    throw new TrError('Service not found in cache while preparing evolutionary transit network design job', 'ALGOCACHE001');
+                }
+                return { service, numberOfVehicles: lvlOfService.numberOfVehicles };
+            });
+            return acc;
+        }, {});
+        this.setLineServices(lineServices);
+    
+        console.timeEnd(`Preparing data for evolutionary transit network design job from cache ${jobId}`);
+    }
+
+    private _run = async (
+
+    ): Promise<boolean> => {
+        // Prepare and load data, either from cache when resuming a started job or from server at first run
+        if (this.job.attributes.internal_data.dataPrepared !== true) {
+            await this._loadAndPrepareData();
+        } else {
+            await this._loadAndPrepareDataFromCache();
         }
+
         const algorithmResults = this.job.attributes.data.results!;
 
-
         let candidates: NetworkCandidate[] = [];
         let previousGeneration: Generation | undefined = undefined;
         // FIXME Add a checkpoint here and cancellation check

packages/transition-backend/src/services/networkDesign/transitNetworkDesign/evolutionary/types.ts

@@ -30,6 +30,8 @@ export type EvolutionaryTransitNetworkDesignJobType = {
     files: { transitDemand: true; nodeWeight: true };
     internal_data: {
         populationSize?: number;
+        dataPrepared?: boolean;
+        lineServices?: { [lineId: string]: { serviceId: string; numberOfVehicles: number; }[] };
     };
 };