New stagnation stopping criterion (#13)

* tmp commit; implementation of stopping crits and other stuff

* test new stopping criterion

* revert check_termination

* clean optimisation_CMA.py

* clean stoppingCriteria.py

* revert simulators.py

* cleanup

* some fixes

---------

Co-authored-by: Jaquier Aurélien Tristan <[email protected]>

Author: AurelienJaquier

bluepyopt/deapext/CMA_MO.py

@@ -30,7 +30,7 @@
 from deap import base
 from deap import cma
 
-from .stoppingCriteria import MaxNGen
+from .stoppingCriteria import MaxNGen, Stagnationv2
 from . import utils
 from . import hype
 
@@ -160,10 +160,13 @@ def __init__(
         self.active = True
         if max_ngen <= 0:
             max_ngen = 100 + 50 * (self.problem_size + 3) ** 2 / numpy.sqrt(
-                self.lambda_
+                lambda_
             )
 
-        self.stopping_conditions = [MaxNGen(max_ngen)]
+        self.stopping_conditions = [
+            MaxNGen(max_ngen),
+            Stagnationv2(lambda_, self.problem_size),
+        ]
 
     def _select(self, candidates):
         """Select the best candidates of the population
@@ -242,6 +245,6 @@ def check_termination(self, gen):
             if c.criteria_met:
                 logger.info(
                     "CMA stopped because of termination criteria: " +
-                    " ".join(c.name)
+                    "" + " ".join(c.name)
                 )
                 self.active = False

bluepyopt/deapext/CMA_SO.py

@@ -31,7 +31,7 @@
 
 from .stoppingCriteria import (
     MaxNGen,
-    Stagnation,
+    Stagnationv2,
     TolHistFun,
     EqualFunVals,
     NoEffectAxis,
@@ -63,17 +63,17 @@ def __init__(
         """Constructor
 
         Args:
-             centroid (list): initial guess used as the starting point of
-             the CMA-ES
-             offspring_size (int): number of offspring individuals in each
-                 generation
-             sigma (float): initial standard deviation of the distribution
-             max_ngen (int): total number of generation to run
-             IndCreator (fcn): function returning an individual of the pop
-             RandIndCreator (fcn): function creating a random individual.
-             map_function (map): function used to map (parallelize) the
-                 evaluation function calls
-             use_scoop (bool): use scoop map for parallel computation
+            centroid (list): initial guess used as the starting point of
+            the CMA-ES
+            offspring_size (int): number of offspring individuals in each
+                generation
+            sigma (float): initial standard deviation of the distribution
+            max_ngen (int): total number of generation to run
+            IndCreator (fcn): function returning an individual of the pop
+            RandIndCreator (fcn): function creating a random individual.
+            map_function (map): function used to map (parallelize) the
+                evaluation function calls
+            use_scoop (bool): use scoop map for parallel computation
         """
 
         if offspring_size is None:
@@ -103,14 +103,14 @@ def __init__(
         self.active = True
         if max_ngen <= 0:
             max_ngen = 100 + 50 * (self.problem_size + 3) ** 2 / numpy.sqrt(
-                self.lambda_
+                lambda_
             )
 
         self.stopping_conditions = [
             MaxNGen(max_ngen),
-            Stagnation(self.lambda_, self.problem_size),
-            TolHistFun(self.lambda_, self.problem_size),
-            EqualFunVals(self.lambda_, self.problem_size),
+            Stagnationv2(lambda_, self.problem_size),
+            TolHistFun(lambda_, self.problem_size),
+            EqualFunVals(lambda_, self.problem_size),
             NoEffectAxis(self.problem_size),
             TolUpSigma(float(self.sigma)),
             TolX(),
@@ -220,6 +220,6 @@ def check_termination(self, gen):
             if c.criteria_met:
                 logger.info(
                     "CMA stopped because of termination criteria: " +
-                    " ".join(c.name)
+                    "" + " ".join(c.name)
                 )
                 self.active = False

bluepyopt/deapext/optimisationsCMA.py

@@ -41,7 +41,6 @@
 def _ind_convert_space(ind, convert_fcn):
     """util function to pass the individual from normalized to real space and
     inversely"""
-
     return [f(x) for f, x in zip(convert_fcn, ind)]
 
 
@@ -359,7 +358,7 @@ def run(
                 pickle.dump(cp, open(cp_filename_tmp, "wb"))
                 if os.path.isfile(cp_filename_tmp):
                     shutil.copy(cp_filename_tmp, cp_filename)
-                    logger.debug('Wrote checkpoint to %s', cp_filename)
+                    logger.debug("Wrote checkpoint to %s", cp_filename)
 
                 CMA_es.map_function = temp_mf
 

bluepyopt/deapext/stoppingCriteria.py

@@ -75,8 +75,10 @@ def check(self, kwargs):
         fitness = [ind.fitness.reduce for ind in population]
         fitness.sort()
 
-        self.best.append(fitness[0])
-        self.median.append(fitness[int(round(len(fitness) / 2.0))])
+        # condition to avoid duplicates when re-starting
+        if len(self.best) < ngen:
+            self.best.append(fitness[0])
+            self.median.append(fitness[int(round(len(fitness) / 2.0))])
         self.stagnation_iter = int(
             numpy.ceil(
                 0.2 * ngen + 120 + 30.0 * self.problem_size / self.lambda_
@@ -95,6 +97,64 @@ def check(self, kwargs):
             self.criteria_met = True
 
 
+class Stagnationv2(bluepyopt.stoppingCriteria.StoppingCriteria):
+    """Stagnation stopping criteria class"""
+
+    name = "Stagnationv2"
+
+    def __init__(
+        self, lambda_, problem_size, threshold=0.01, std_threshold=0.02
+    ):
+        """Constructor
+        
+        Args:
+            lambda_ (int): offspring size
+            problem_size (int): problem size
+            threshold (float): 1st criterion is triggered if best fitness
+                improves less than this threshold for 100 generations
+            std_threshold (float): 2nd criterion is triggered if
+                standard deviation of the best fitness over
+                the last 20 generations is below the best fitness multiplied
+                by this threshold
+        """
+        super(Stagnationv2, self).__init__()
+
+        self.lambda_ = lambda_
+        self.problem_size = problem_size
+        self.stagnation_iter = int(
+            numpy.ceil(
+                0.2 * ngen + 120 + 30.0 * self.problem_size / self.lambda_
+            )
+        )
+        self.threshold = threshold
+        self.std_threshold = std_threshold
+
+        self.best = []
+
+    def check(self, kwargs):
+        """Check if best model fitness does not improve over 1% over 100 gens
+            and is not noisy in the last 20 generations
+        """
+        ngen = kwargs.get("gen")
+        population = kwargs.get("population")
+        fitness = [ind.fitness.reduce for ind in population]
+        fitness.sort()
+
+        # condition to avoid duplicates when re-starting
+        if len(self.best) < ngen:
+            self.best.append(fitness[0])
+
+        crit1 = len(self.best) > self.stagnation_iter
+        crit2 = numpy.median(self.best[-20:]) * (1 + self.threshold) \
+            > numpy.median(self.best[-120:-100])
+        crit3 = numpy.std(self.best[-20:]) < (
+            self.std_threshold * self.best[-1]
+        )
+
+        if crit1 and crit2 and crit3:
+            self.criteria_met = True
+
+
 class TolHistFun(bluepyopt.stoppingCriteria.StoppingCriteria):
     """TolHistFun stopping criteria class"""
 
@@ -105,8 +165,7 @@ def __init__(self, lambda_, problem_size):
         super(TolHistFun, self).__init__()
         self.tolhistfun = 10 ** -12
         self.mins = deque(
-            maxlen=10 + int(numpy.ceil(30.0 * problem_size / lambda_))
-        )
+            maxlen=10 + int(numpy.ceil(30.0 * problem_size / lambda_)))
 
     def check(self, kwargs):
         """Check if the range of the best values is smaller than