@@ -36,10 +36,18 @@ class Optimizer(object):
36
36
"""
37
37
38
38
def __init__ (self , learning_rate , global_step = None , regularization = None ):
39
- assert learning_rate is not None
39
+ if not isinstance (learning_rate , float ) and \
40
+ not isinstance (learning_rate , framework .Variable ):
41
+ raise TypeError ("learning rate should be float or Variable" )
40
42
self ._global_step = global_step
41
43
self .regularization = regularization
42
- self ._global_learning_rate = learning_rate
44
+ self ._learning_rate = learning_rate
45
+ # each program should have a independent learning rate
46
+ # program -> Variable(learning_rate)
47
+ self ._learning_rate_map = dict ()
48
+ if isinstance (self ._learning_rate , framework .Variable ):
49
+ self ._learning_rate_map [framework .default_main_program (
50
+ )] = self ._learning_rate
43
51
# Dictionary of accumulators. Some optimizer subclasses need to
44
52
# allocate and manage extra variables associated with the parameters
45
53
# to train. These variables are called accumulators.
@@ -48,26 +56,33 @@ def __init__(self, learning_rate, global_step=None, regularization=None):
48
56
self .helper = None
49
57
50
58
def _create_global_learning_rate (self ):
51
- if isinstance (self ._global_learning_rate , float ):
52
- self ._global_learning_rate = layers .create_global_var (
53
- name = unique_name .generate ("learning_rate" ),
54
- shape = [1 ],
55
- value = float (self ._global_learning_rate ),
56
- dtype = 'float32' ,
57
- persistable = True )
58
-
59
- if not isinstance (self ._global_learning_rate , framework .Variable ):
60
- raise ValueError ("learning rate should be a Variable, "
61
- "actual type is %s" ,
62
- type (self ._global_learning_rate ))
63
-
64
- @property
65
- def global_learning_rate (self ):
59
+ lr = self .global_learning_rate ()
60
+
61
+ if isinstance (lr , framework .Variable ):
62
+ return
63
+ else :
64
+ if not isinstance (self ._learning_rate , float ):
65
+ raise TypeError (
66
+ "learning rate variable is create outside optimizer,"
67
+ "can not create new learning rate variable for new program" )
68
+
69
+ # create learning rate in the current main program
70
+ self ._learning_rate_map [framework .default_main_program (
71
+ )] = layers .create_global_var (
72
+ name = unique_name .generate ("learning_rate" ),
73
+ shape = [1 ],
74
+ value = float (self ._learning_rate ),
75
+ dtype = 'float32' ,
76
+ persistable = True )
77
+
78
+ def global_learning_rate (self , program = None ):
66
79
"""
67
80
get global decayed learning rate
68
81
:return:
69
82
"""
70
- return self ._global_learning_rate
83
+ if program is None :
84
+ program = framework .default_main_program ()
85
+ return self ._learning_rate_map .get (program , None )
71
86
72
87
def _append_optimize_op (self , block , param_and_grad ):
73
88
""" append optimize operator to block and return all the added optimize_op
@@ -78,7 +93,7 @@ def _create_param_lr(self, param_and_grad):
78
93
# create learning rate variable for every parameter
79
94
param = param_and_grad [0 ]
80
95
param_lr = param .optimize_attr ['learning_rate' ]
81
- return self ._global_learning_rate * param_lr
96
+ return self .global_learning_rate () * param_lr
82
97
83
98
def _create_accumulators (self , block , parameters ):
84
99
"""Create all accumulators needed by the parameters
0 commit comments