Merge pull request #460 from BlueBrain/save-frequency

possibility of using a minimum amount of time before saving optimisation

Author: AurelienJaquier

bluepyopt/deapext/algorithms.py

@@ -26,6 +26,7 @@
 import logging
 import shutil
 import os
+import time
 
 import deap.algorithms
 import deap.tools
@@ -92,6 +93,7 @@ def eaAlphaMuPlusLambdaCheckpoint(
         stats=None,
         halloffame=None,
         cp_frequency=1,
+        cp_period=None,
         cp_filename=None,
         continue_cp=False,
         terminator=None,
@@ -108,6 +110,8 @@ def eaAlphaMuPlusLambdaCheckpoint(
         stats(deap.tools.Statistics): generation of statistics
         halloffame(deap.tools.HallOfFame): hall of fame
         cp_frequency(int): generations between checkpoints
+        cp_period(float): minimum time (in s) between checkpoint.
+            None to save checkpoint independently of the time between them
         cp_filename(string): path to checkpoint filename
         continue_cp(bool): whether to continue
         terminator (multiprocessing.Event): exit loop when is set.
@@ -159,6 +163,7 @@ def eaAlphaMuPlusLambdaCheckpoint(
     # Begin the generational process
     gen = start_gen + 1
     stopping_params = {"gen": gen}
+    time_last_save = time.time()
     while utils.run_next_gen(
             not (_check_stopping_criteria(stopping_criteria, stopping_params)),
             terminator):
@@ -176,7 +181,8 @@ def eaAlphaMuPlusLambdaCheckpoint(
         logger.info(logbook.stream)
 
         if (cp_filename and cp_frequency and
-           gen % cp_frequency == 0):
+           gen % cp_frequency == 0 and
+           (cp_period is None or time.time() - time_last_save > cp_period)):
             cp = dict(population=population,
                       generation=gen,
                       parents=parents,
@@ -190,6 +196,8 @@ def eaAlphaMuPlusLambdaCheckpoint(
                 shutil.copy(cp_filename_tmp, cp_filename)
                 logger.debug('Wrote checkpoint to %s', cp_filename)
 
+            time_last_save = time.time()
+
         gen += 1
         stopping_params["gen"] = gen
 

bluepyopt/deapext/optimisations.py

@@ -256,6 +256,7 @@ def run(self,
             continue_cp=False,
             cp_filename=None,
             cp_frequency=1,
+            cp_period=None,
             parent_population=None,
             terminator=None):
         """Run optimisation"""
@@ -315,6 +316,7 @@ def run(self,
             stats=stats,
             halloffame=self.hof,
             cp_frequency=cp_frequency,
+            cp_period=None,
             continue_cp=continue_cp,
             cp_filename=cp_filename,
             terminator=terminator,

bluepyopt/deapext/optimisationsCMA.py

@@ -26,6 +26,7 @@
 import functools
 import shutil
 import os
+import time
 
 import deap.tools
 
@@ -243,6 +244,7 @@ def run(
         self,
         max_ngen=0,
         cp_frequency=1,
+        cp_period=None,
         continue_cp=False,
         cp_filename=None,
         terminator=None,
@@ -252,6 +254,8 @@ def run(
         Args:
             max_ngen(int): Total number of generation to run
             cp_frequency(int): generations between checkpoints
+            cp_period(float): minimum time (in s) between checkpoint.
+                None to save checkpoint independently of the time between them
             continue_cp(bool): whether to continue
             cp_filename(string): path to checkpoint filename
             terminator (multiprocessing.Event): exit loop when is set.
@@ -309,6 +313,7 @@ def run(
         if hasattr(self.evaluator, "param_names"):
             param_names = self.evaluator.param_names
 
+        time_last_save = time.time()
         # Run until a termination criteria is met
         while utils.run_next_gen(CMA_es.active, terminator):
             logger.info("Generation {}".format(gen))
@@ -343,8 +348,12 @@ def run(
             # termination conditions were reached
             CMA_es.update_strategy()
             CMA_es.check_termination(gen)
-
-            if cp_filename and cp_frequency and gen % cp_frequency == 0:
+            if (
+                cp_filename and
+                cp_frequency and
+                gen % cp_frequency == 0 and
+                (cp_period is None or time.time() - time_last_save > cp_period)
+            ):
 
                 # Map function shouldn't be pickled
                 temp_mf = CMA_es.map_function
@@ -368,6 +377,8 @@ def run(
 
                 CMA_es.map_function = temp_mf
 
+                time_last_save = time.time()
+
             gen += 1
 
         return pop, self.hof, logbook, history