Skip to content

Commit ed1a053

Browse files
authored
Merge pull request #8545 from jacquesqiao/fix-optimize-multi-program
create learning rate for multi program
2 parents 2a3b9ff + ea9e62b commit ed1a053

File tree

1 file changed

+34
-19
lines changed

1 file changed

+34
-19
lines changed

python/paddle/fluid/optimizer.py

Lines changed: 34 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,18 @@ class Optimizer(object):
3636
"""
3737

3838
def __init__(self, learning_rate, global_step=None, regularization=None):
39-
assert learning_rate is not None
39+
if not isinstance(learning_rate, float) and \
40+
not isinstance(learning_rate, framework.Variable):
41+
raise TypeError("learning rate should be float or Variable")
4042
self._global_step = global_step
4143
self.regularization = regularization
42-
self._global_learning_rate = learning_rate
44+
self._learning_rate = learning_rate
45+
# each program should have a independent learning rate
46+
# program -> Variable(learning_rate)
47+
self._learning_rate_map = dict()
48+
if isinstance(self._learning_rate, framework.Variable):
49+
self._learning_rate_map[framework.default_main_program(
50+
)] = self._learning_rate
4351
# Dictionary of accumulators. Some optimizer subclasses need to
4452
# allocate and manage extra variables associated with the parameters
4553
# to train. These variables are called accumulators.
@@ -48,26 +56,33 @@ def __init__(self, learning_rate, global_step=None, regularization=None):
4856
self.helper = None
4957

5058
def _create_global_learning_rate(self):
51-
if isinstance(self._global_learning_rate, float):
52-
self._global_learning_rate = layers.create_global_var(
53-
name=unique_name.generate("learning_rate"),
54-
shape=[1],
55-
value=float(self._global_learning_rate),
56-
dtype='float32',
57-
persistable=True)
58-
59-
if not isinstance(self._global_learning_rate, framework.Variable):
60-
raise ValueError("learning rate should be a Variable, "
61-
"actual type is %s",
62-
type(self._global_learning_rate))
63-
64-
@property
65-
def global_learning_rate(self):
59+
lr = self.global_learning_rate()
60+
61+
if isinstance(lr, framework.Variable):
62+
return
63+
else:
64+
if not isinstance(self._learning_rate, float):
65+
raise TypeError(
66+
"learning rate variable is create outside optimizer,"
67+
"can not create new learning rate variable for new program")
68+
69+
# create learning rate in the current main program
70+
self._learning_rate_map[framework.default_main_program(
71+
)] = layers.create_global_var(
72+
name=unique_name.generate("learning_rate"),
73+
shape=[1],
74+
value=float(self._learning_rate),
75+
dtype='float32',
76+
persistable=True)
77+
78+
def global_learning_rate(self, program=None):
6679
"""
6780
get global decayed learning rate
6881
:return:
6982
"""
70-
return self._global_learning_rate
83+
if program is None:
84+
program = framework.default_main_program()
85+
return self._learning_rate_map.get(program, None)
7186

7287
def _append_optimize_op(self, block, param_and_grad):
7388
""" append optimize operator to block and return all the added optimize_op
@@ -78,7 +93,7 @@ def _create_param_lr(self, param_and_grad):
7893
# create learning rate variable for every parameter
7994
param = param_and_grad[0]
8095
param_lr = param.optimize_attr['learning_rate']
81-
return self._global_learning_rate * param_lr
96+
return self.global_learning_rate() * param_lr
8297

8398
def _create_accumulators(self, block, parameters):
8499
"""Create all accumulators needed by the parameters

0 commit comments

Comments
 (0)