networkDesign backend: checkpoint generation data

tahini · tahini · commit 2a054d35c3a5 · 2025-12-01T13:38:32.000-05:00
After the scenarios are prepared for a generation, add a checkpoint for
the generation, saving the candidate chromosomes and scenarioId.

Upon resuming, if the previous generation is undefined and there is
checkoint data, recover the previous candidates, as well as their
corresponding scenarios from cache.

For now, the whole generation will be re-calculated, the candidates's
result are not yet saved as checkpoints.
diff --git a/packages/transition-backend/src/services/evolutionaryAlgorithm/candidate/LineAndNumberOfVehiclesNetworkCandidate.ts b/packages/transition-backend/src/services/evolutionaryAlgorithm/candidate/LineAndNumberOfVehiclesNetworkCandidate.ts
@@ -28,9 +28,11 @@ class LineAndNumberOfVehiclesNetworkCandidate extends Candidate {
 
     constructor(
         chromosome: AlgoTypes.CandidateChromosome,
-        wrappedJob: TransitNetworkDesignJobWrapper<EvolutionaryTransitNetworkDesignJobType>
+        wrappedJob: TransitNetworkDesignJobWrapper<EvolutionaryTransitNetworkDesignJobType>,
+        scenario?: Scenario
     ) {
         super(chromosome, wrappedJob);
+        this.scenario = scenario;
     }
 
     private prepareNetwork(): Line[] {
diff --git a/packages/transition-backend/src/services/evolutionaryAlgorithm/generation/LineAndNumberOfVehiclesGeneration.ts b/packages/transition-backend/src/services/evolutionaryAlgorithm/generation/LineAndNumberOfVehiclesGeneration.ts
@@ -21,6 +21,9 @@ import LineAndNumberOfVehiclesGenerationLogger from './LineAndNumberOfVehiclesGe
 import { EvolutionaryTransitNetworkDesignJob, EvolutionaryTransitNetworkDesignJobType } from '../../networkDesign/transitNetworkDesign/evolutionary/types';
 import LineCollection from 'transition-common/lib/services/line/LineCollection';
 import { TransitNetworkDesignJobWrapper } from '../../networkDesign/transitNetworkDesign/TransitNetworkDesignJobWrapper';
+import ScenarioCollection from 'transition-common/lib/services/scenario/ScenarioCollection';
+import Candidate from '../candidate/Candidate';
+import LineAndNumberOfVehiclesNetworkCandidate from '../candidate/LineAndNumberOfVehiclesNetworkCandidate';
 
 const chromosomeExists = (chrom: boolean[], linesChromosomes: boolean[][]) =>
     linesChromosomes.findIndex((chromosome) => _isEqual(chromosome, chrom)) !== -1;
@@ -226,6 +229,20 @@ export const reproduceCandidates = (
     return candidates;
 };
 
+export const resumeCandidatesFromChromosomes = (
+    jobWrapper: TransitNetworkDesignJobWrapper<EvolutionaryTransitNetworkDesignJobType>,
+    currentGeneration: Exclude<EvolutionaryTransitNetworkDesignJobType['internal_data']['currentGeneration'], undefined>,
+    scenarioCollection: ScenarioCollection
+): NetworkCandidate[] => {
+    return currentGeneration.candidates.map((candidateData) => 
+        new NetworkCandidate(
+            candidateData.chromosome,
+            jobWrapper,
+            scenarioCollection.getById(candidateData.scenarioId!)
+        )
+    );
+};
+
 /**
  * Represents the current generation of the simulation
  */
@@ -247,6 +264,10 @@ class LineAndNumberOfVehiclesGeneration extends Generation {
         );
     }
 
+    getCandidates(): LineAndNumberOfVehiclesNetworkCandidate[] {
+        return this.candidates as LineAndNumberOfVehiclesNetworkCandidate[];
+    }
+
     // TODO Sorting candidates and calculating totalFitness maybe should not be
     // part of the evolutionary algorithm code, but we may need to extract some
     // result types. For now, it's the only algorithm we have, so keep it here
diff --git a/packages/transition-backend/src/services/networkDesign/transitNetworkDesign/evolutionary/EvolutionaryTransitNetworkDesignJob.ts b/packages/transition-backend/src/services/networkDesign/transitNetworkDesign/evolutionary/EvolutionaryTransitNetworkDesignJob.ts
@@ -22,12 +22,14 @@ import { ResultSerialization } from '../../../evolutionaryAlgorithm/candidate/Ca
 import NetworkCandidate from '../../../evolutionaryAlgorithm/candidate/LineAndNumberOfVehiclesNetworkCandidate';
 import Generation from '../../../evolutionaryAlgorithm/generation/Generation';
 import * as AlgoTypes from '../../../evolutionaryAlgorithm/internalTypes';
-import LineAndNumberOfVehiclesGeneration, { generateFirstCandidates, reproduceCandidates } from '../../../evolutionaryAlgorithm/generation/LineAndNumberOfVehiclesGeneration';
+import LineAndNumberOfVehiclesGeneration, { generateFirstCandidates, reproduceCandidates, resumeCandidatesFromChromosomes } from '../../../evolutionaryAlgorithm/generation/LineAndNumberOfVehiclesGeneration';
 import TrError from 'chaire-lib-common/lib/utils/TrError';
 import { collectionToCache as serviceCollectionToCache, collectionFromCache as serviceCollectionFromCache } from '../../../../models/capnpCache/transitServices.cache.queries';
-import { objectsToCache as linesToCache, objectFromCache as lineFromCache } from '../../../../models/capnpCache/transitLines.cache.queries';
+import { collectionFromCache as scenarioCollectionFromCache } from '../../../../models/capnpCache/transitScenarios.cache.queries';
+import { objectsToCache as linesToCache, objectFromCache as lineFromCache, collectionToCache as lineCollectionToCache, collectionFromCache as lineCollectionFromCache } from '../../../../models/capnpCache/transitLines.cache.queries';
 import { prepareServices, saveSimulationScenario } from '../../../evolutionaryAlgorithm/preparation/ServicePreparation';
 import Line from 'transition-common/lib/services/line/Line';
+import ScenarioCollection from 'transition-common/lib/services/scenario/ScenarioCollection';
 
 /**
  * Do batch calculation on a csv file input
@@ -153,6 +155,10 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
         await serviceCollectionToCache(services, cacheDirectoryPath);
         console.log(`Saved service cache file in ${cacheDirectoryPath}.`);
         await linesToCache(simulatedLineCollection.getFeatures(), cacheDirectoryPath);
+        // Save the line collection in a special cache directory. The order of
+        // the lines is very important for the chromosome and we should make
+        // sure it remains the same upon resume of the Job.
+        await lineCollectionToCache(simulatedLineCollection, cacheDirectoryPath + '/simulatedLines');
         console.log(`Saved lines cache files in ${cacheDirectoryPath}.`);
 
         // Add non simulated services to the collection, ie those used by the simulation, but not generated by it
@@ -227,8 +233,8 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
         // FIXME Do we even need this? Or can we just start recovery at serviceCollectionFromCache call and get only required data?
         await this.loadServerData(serviceLocator.socketEventManager);
         
-        // Get the simulated lines
-        const simulatedLineCollection = this._getSimulatedLineCollection({ agencies: this.agencyCollection, lines: this.allLineCollection, services: this.serviceCollection });
+        // Get the simulated lines from cache, to make sure the order is the same as before
+        const simulatedLineCollection = await lineCollectionFromCache(this.getCacheDirectory() + '/simulatedLines');
         this.simulatedLineCollection = simulatedLineCollection;
 
         // Read the services from the cache, with all individual line services
@@ -254,10 +260,48 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
             return acc;
         }, {});
         this.setLineServices(lineServices);
-    
+
         console.timeEnd(`Preparing data for evolutionary transit network design job from cache ${jobId}`);
     }
 
+    private _prepareOrResumeGeneration = async (previousGeneration: LineAndNumberOfVehiclesGeneration | undefined): Promise<LineAndNumberOfVehiclesGeneration> => {
+        // Resume candidates from checkpoint if possible, scenarios are prepared
+        if (previousGeneration === undefined && this.job.attributes.internal_data.checkpoint !== undefined && this.job.attributes.internal_data.checkpoint > 0 && this.job.attributes.internal_data.currentGeneration !== undefined) {
+            this.currentIteration = this.job.attributes.internal_data.checkpoint;
+            const currentGenerationData = this.job.attributes.internal_data.currentGeneration;
+            // Get the current scenario collection from cache to recover previous scenarios
+            const scenarioCollection = await scenarioCollectionFromCache(this.getCacheDirectory());
+            const candidates = resumeCandidatesFromChromosomes(this, currentGenerationData, scenarioCollection);
+            console.log(`Resumed generation ${this.currentIteration} with ${candidates.length} candidates from checkpoint.`);
+            return new LineAndNumberOfVehiclesGeneration(
+                candidates,
+                this,
+                this.currentIteration
+            );
+        }
+        // Generate or reproduce candidates
+        const candidates = this.currentIteration === 1 || previousGeneration === undefined
+            ? generateFirstCandidates(this) 
+            : reproduceCandidates(this, previousGeneration.getCandidates(), this.currentIteration)
+        const currentGeneration = new LineAndNumberOfVehiclesGeneration(
+            candidates,
+            this,
+            this.currentIteration
+        );
+        const messages = await currentGeneration.prepareCandidates(this.executorOptions.progressEmitter);
+        await this.addMessages(messages);
+        // Add a checkpoint for the current generation
+        this.job.attributes.internal_data.checkpoint = this.currentIteration;
+        this.job.attributes.internal_data.currentGeneration = {
+            candidates: candidates.map((candidate) => ({
+                chromosome: candidate.getChromosome(),
+                scenarioId: candidate.getScenario()?.id
+            }))
+        }
+        await this.job.save(this.executorOptions.progressEmitter);
+        return currentGeneration;
+    }
+
     private _run = async (
 
     ): Promise<boolean> => {
@@ -270,23 +314,13 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
 
         const algorithmResults = this.job.attributes.data.results!;
         
-        let candidates: NetworkCandidate[] = [];
-        let previousGeneration: Generation | undefined = undefined;
-        // FIXME Add a checkpoint here and cancellation check
+        let previousGeneration: LineAndNumberOfVehiclesGeneration | undefined = undefined;
         try {
-            // FIXME Recover from checkpoint
-            while (!this.isFinished()) {
-                candidates =
-                    this.currentIteration === 1
-                        ? generateFirstCandidates(this)
-                        : reproduceCandidates(this, candidates, this.currentIteration);
-                previousGeneration = new LineAndNumberOfVehiclesGeneration(
-                    candidates,
-                    this,
-                    this.currentIteration
-                );
-                const messages = await previousGeneration.prepareCandidates(this.executorOptions.progressEmitter);
-                await this.addMessages(messages);
+            do {
+                previousGeneration = await this._prepareOrResumeGeneration(previousGeneration);
+
+                // FIXME Handle checkpointing with simulations that can be long
+                // Simulate the generation
                 await previousGeneration.simulate();
                 // TODO For now, we keep the best of each generation, but we should
                 // be smarter about it, knowing that the end of the simulation can
@@ -312,7 +346,7 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
                     this.job.attributes.data.results = algorithmResults;
                     await this.job.save(this.executorOptions.progressEmitter);
                 }
-            }
+            } while (!this.isFinished() && !this.executorOptions.isCancelled());
         } catch(error) {
             console.log('error during evolutionary algorithm generations', error);
             throw error;
@@ -323,9 +357,7 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
             throw new TrError('Evolutionary Algorithm: no generation was done!', 'ALGOGEN001');
         }
 
-        console.log('Evolutionary transit network design job completed.');
-        this.job.setCompleted();
-        await this.job.save(this.executorOptions.progressEmitter);
+        
 
         return true;
     };
@@ -335,6 +367,19 @@ class EvolutionaryTransitNetworkDesignJobExecutor extends TransitNetworkDesignJo
         try {
             
             const result = await this._run();
+
+            if (this.executorOptions.isCancelled()) {
+                console.log('Evolutionary transit network design job cancelled.');
+                return {
+                    status: 'paused',
+                    warnings: [],
+                    errors: []
+                };
+            }
+
+            console.log('Evolutionary transit network design job completed.');
+            this.job.setCompleted();
+            await this.job.save(this.executorOptions.progressEmitter);
            
             return {
                 status: 'success',
diff --git a/packages/transition-backend/src/services/networkDesign/transitNetworkDesign/evolutionary/types.ts b/packages/transition-backend/src/services/networkDesign/transitNetworkDesign/evolutionary/types.ts
@@ -10,6 +10,7 @@ import { AlgorithmConfigurationByType } from 'transition-common/lib/services/net
 import { SimulationMethodConfiguration } from 'transition-common/lib/services/networkDesign/transit/simulationMethod';
 import { ExecutableJob } from '../../../executableJob/ExecutableJob';
 import { ResultSerialization } from '../../../evolutionaryAlgorithm/candidate/Candidate';
+import { CandidateChromosome } from '../../../evolutionaryAlgorithm/internalTypes';
 
 export type EvolutionaryTransitNetworkDesignJobParameters = {
     transitNetworkDesignParameters: TransitNetworkDesignParameters;
@@ -32,6 +33,13 @@ export type EvolutionaryTransitNetworkDesignJobType = {
         populationSize?: number;
         dataPrepared?: boolean;
         lineServices?: { [lineId: string]: { serviceId: string; numberOfVehicles: number; }[] };
+        currentGeneration?: {
+            candidates: {
+                chromosome: CandidateChromosome;
+                scenarioId?: string;
+                fitness?: number;
+            }[]
+        }
     };
 };
 
