1- from tensorflow .keras .callbacks import *
2- from tensorflow .keras import backend as K
31import numpy as np
2+ from tensorflow .keras import backend as K
3+ from tensorflow .keras .callbacks import *
4+
45
56class CyclicLR (Callback ):
67 """This callback implements a cyclical learning rate policy (CLR).
78 The method cycles the learning rate between two boundaries with
89 some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).
9- The amplitude of the cycle can be scaled on a per-iteration or
10+ The amplitude of the cycle can be scaled on a per-iteration or
1011 per-cycle basis.
1112 This class has three built-in policies, as put forth in the paper.
1213 "triangular":
1314 A basic triangular cycle w/ no amplitude scaling.
1415 "triangular2":
1516 A basic triangular cycle that scales initial amplitude by half each cycle.
1617 "exp_range":
17- A cycle that scales initial amplitude by gamma**(cycle iterations) at each
18+ A cycle that scales initial amplitude by gamma**(cycle iterations) at each
1819 cycle iteration.
1920 For more detail, please see paper.
20-
21+
2122 # Example
2223 ```python
2324 clr = CyclicLR(base_lr=0.001, max_lr=0.006,
2425 step_size=2000., mode='triangular')
2526 model.fit(X_train, Y_train, callbacks=[clr])
2627 ```
27-
28+
2829 Class also supports custom scaling functions:
2930 ```python
3031 clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))
3132 clr = CyclicLR(base_lr=0.001, max_lr=0.006,
3233 step_size=2000., scale_fn=clr_fn,
3334 scale_mode='cycle')
3435 model.fit(X_train, Y_train, callbacks=[clr])
35- ```
36+ ```
3637 # Arguments
3738 base_lr: initial learning rate which is the
3839 lower boundary in the cycle.
3940 max_lr: upper boundary in the cycle. Functionally,
4041 it defines the cycle amplitude (max_lr - base_lr).
4142 The lr at any cycle is the sum of base_lr
42- and some scaling of the amplitude; therefore
43+ and some scaling of the amplitude; therefore
4344 max_lr may not actually be reached depending on
4445 scaling function.
4546 step_size: number of training iterations per
@@ -52,17 +53,25 @@ class CyclicLR(Callback):
5253 gamma: constant in 'exp_range' scaling function:
5354 gamma**(cycle iterations)
5455 scale_fn: Custom scaling policy defined by a single
55- argument lambda function, where
56+ argument lambda function, where
5657 0 <= scale_fn(x) <= 1 for all x >= 0.
57- mode paramater is ignored
58+ mode paramater is ignored
5859 scale_mode: {'cycle', 'iterations'}.
59- Defines whether scale_fn is evaluated on
60+ Defines whether scale_fn is evaluated on
6061 cycle number or cycle iterations (training
6162 iterations since start of cycle). Default is 'cycle'.
6263 """
6364
64- def __init__ (self , base_lr = 0.001 , max_lr = 0.006 , step_size = 2000. , mode = 'triangular' ,
65- gamma = 1. , scale_fn = None , scale_mode = 'cycle' ):
65+ def __init__ (
66+ self ,
67+ base_lr = 0.001 ,
68+ max_lr = 0.006 ,
69+ step_size = 2000.0 ,
70+ mode = "triangular" ,
71+ gamma = 1.0 ,
72+ scale_fn = None ,
73+ scale_mode = "cycle" ,
74+ ):
6675 super (CyclicLR , self ).__init__ ()
6776
6877 self .base_lr = base_lr
@@ -71,26 +80,25 @@ def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangula
7180 self .mode = mode
7281 self .gamma = gamma
7382 if scale_fn == None :
74- if self .mode == ' triangular'
75- self .scale_fn = lambda x : 1.
76- self .scale_mode = ' cycle'
77- elif self .mode == ' triangular2'
78- self .scale_fn = lambda x : 1 / (2. ** ( x - 1 ))
79- self .scale_mode = ' cycle'
80- elif self .mode == ' exp_range'
81- self .scale_fn = lambda x : gamma ** (x )
82- self .scale_mode = ' iterations'
83+ if self .mode == " triangular"
84+ self .scale_fn = lambda x : 1.0
85+ self .scale_mode = " cycle"
86+ elif self .mode == " triangular2"
87+ self .scale_fn = lambda x : 1 / (2.0 ** ( x - 1 ))
88+ self .scale_mode = " cycle"
89+ elif self .mode == " exp_range"
90+ self .scale_fn = lambda x : gamma ** (x )
91+ self .scale_mode = " iterations"
8392 else :
8493 self .scale_fn = scale_fn
8594 self .scale_mode = scale_mode
86- self .clr_iterations = 0.
87- self .trn_iterations = 0.
95+ self .clr_iterations = 0.0
96+ self .trn_iterations = 0.0
8897 self .history = {}
8998
9099 self ._reset ()
91100
92- def _reset (self , new_base_lr = None , new_max_lr = None ,
93- new_step_size = None ):
101+ def _reset (self , new_base_lr = None , new_max_lr = None , new_step_size = None ):
94102 """Resets cycle iterations.
95103 Optional boundary/step size adjustment.
96104 """
@@ -100,34 +108,38 @@ def _reset(self, new_base_lr=None, new_max_lr=None,
100108 self .max_lr = new_max_lr
101109 if new_step_size != None :
102110 self .step_size = new_step_size
103- self .clr_iterations = 0.
104-
111+ self .clr_iterations = 0.0
112+
105113 def clr (self ):
106- cycle = np .floor (1 + self .clr_iterations / (2 * self .step_size ))
107- x = np .abs (self .clr_iterations / self .step_size - 2 * cycle + 1 )
108- if self .scale_mode == 'cycle' :
109- return self .base_lr + (self .max_lr - self .base_lr )* np .maximum (0 , (1 - x ))* self .scale_fn (cycle )
114+ cycle = np .floor (1 + self .clr_iterations / (2 * self .step_size ))
115+ x = np .abs (self .clr_iterations / self .step_size - 2 * cycle + 1 )
116+ if self .scale_mode == "cycle" :
117+ return self .base_lr + (self .max_lr - self .base_lr ) * np .maximum (
118+ 0 , (1 - x )
119+ ) * self .scale_fn (cycle )
110120 else :
111- return self .base_lr + (self .max_lr - self .base_lr )* np .maximum (0 , (1 - x ))* self .scale_fn (self .clr_iterations )
112-
121+ return self .base_lr + (self .max_lr - self .base_lr ) * np .maximum (
122+ 0 , (1 - x )
123+ ) * self .scale_fn (self .clr_iterations )
124+
113125 def on_train_begin (self , logs = {}):
114126 logs = logs or {}
115127
116128 if self .clr_iterations == 0 :
117129 K .set_value (self .model .optimizer .lr , self .base_lr )
118130 else :
119- K .set_value (self .model .optimizer .lr , self .clr ())
120-
131+ K .set_value (self .model .optimizer .lr , self .clr ())
132+
121133 def on_batch_end (self , epoch , logs = None ):
122-
134+
123135 logs = logs or {}
124136 self .trn_iterations += 1
125137 self .clr_iterations += 1
126138
127- self .history .setdefault ('lr' , []).append (K .get_value (self .model .optimizer .lr ))
128- self .history .setdefault (' iterations' append (self .trn_iterations )
139+ self .history .setdefault ("lr" , []).append (K .get_value (self .model .optimizer .lr ))
140+ self .history .setdefault (" iterations" append (self .trn_iterations )
129141
130142 for k , v in logs .items ():
131143 self .history .setdefault (k , []).append (v )
132-
144+
133145 K .set_value (self .model .optimizer .lr , self .clr ())
0 commit comments