Skip to content

Commit f04ec6b

Browse files
committed
Add test_weight_decay
test=release/1.2
1 parent 3abb2aa commit f04ec6b

File tree

1 file changed

+188
-0
lines changed

1 file changed

+188
-0
lines changed
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from __future__ import print_function
16+
import contextlib
17+
18+
import unittest
19+
from functools import partial
20+
import numpy as np
21+
import paddle
22+
import paddle.fluid.core as core
23+
24+
import paddle.fluid as fluid
25+
26+
27+
def get_places():
28+
places = []
29+
if core.is_compiled_with_cuda():
30+
places.append(core.CUDAPlace(0))
31+
return places
32+
33+
34+
@contextlib.contextmanager
35+
def prog_scope_guard(main_prog, startup_prog):
36+
scope = fluid.core.Scope()
37+
with fluid.unique_name.guard():
38+
with fluid.scope_guard(scope):
39+
with fluid.program_guard(main_prog, startup_prog):
40+
yield
41+
42+
43+
def bow_net(data,
44+
label,
45+
dict_dim,
46+
is_sparse=False,
47+
emb_dim=128,
48+
hid_dim=128,
49+
hid_dim2=96,
50+
class_dim=2):
51+
"""
52+
BOW net
53+
This model is from https://github.com/PaddlePaddle/models:
54+
fluid/PaddleNLP/text_classification/nets.py
55+
"""
56+
emb = fluid.layers.embedding(
57+
input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim])
58+
bow = fluid.layers.sequence_pool(input=emb, pool_type='sum')
59+
bow_tanh = fluid.layers.tanh(bow)
60+
fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh")
61+
fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh")
62+
prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax")
63+
cost = fluid.layers.cross_entropy(input=prediction, label=label)
64+
avg_cost = fluid.layers.mean(x=cost)
65+
66+
return avg_cost
67+
68+
69+
class TestWeightDecay(unittest.TestCase):
70+
def setUp(self):
71+
self.word_dict = paddle.dataset.imdb.word_dict()
72+
reader = paddle.batch(
73+
paddle.dataset.imdb.train(self.word_dict), batch_size=4)()
74+
self.train_data = [next(reader) for _ in range(5)]
75+
self.learning_rate = .5
76+
77+
def run_executor(self, place, feed_list, loss):
78+
exe = fluid.Executor(place)
79+
feeder = fluid.DataFeeder(feed_list=feed_list, place=place)
80+
exe.run(fluid.default_startup_program())
81+
main_prog = fluid.default_main_program()
82+
loss_set = []
83+
for data in self.train_data:
84+
out = exe.run(main_prog,
85+
feed=feeder.feed(data),
86+
fetch_list=[loss.name])
87+
88+
print("loss %s" % (np.average(out)))
89+
loss_set.append(np.average(out))
90+
91+
return loss_set
92+
93+
def run_parallel_exe(self,
94+
place,
95+
feed_list,
96+
loss,
97+
use_cuda=True,
98+
use_reduce=False,
99+
use_fast_executor=False,
100+
use_ir_memory_optimize=False):
101+
exe = fluid.Executor(place)
102+
feeder = fluid.DataFeeder(feed_list=feed_list, place=place)
103+
exe.run(fluid.default_startup_program())
104+
105+
exec_strategy = fluid.ExecutionStrategy()
106+
if use_fast_executor:
107+
exec_strategy.use_experimental_executor = True
108+
109+
build_strategy = fluid.BuildStrategy()
110+
build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce \
111+
if use_reduce else fluid.BuildStrategy.ReduceStrategy.AllReduce
112+
build_strategy.memory_optimize = use_ir_memory_optimize
113+
114+
parallel_exe = fluid.ParallelExecutor(
115+
use_cuda,
116+
loss_name=loss.name,
117+
exec_strategy=exec_strategy,
118+
build_strategy=build_strategy)
119+
120+
loss_set = []
121+
for data in self.train_data:
122+
out = parallel_exe.run(feed=feeder.feed(data),
123+
fetch_list=[loss.name])
124+
print("loss %s" % (np.average(out)))
125+
loss_set.append(np.average(out))
126+
127+
return loss_set
128+
129+
def check_weight_decay(self,
130+
place,
131+
model,
132+
use_parallel_exe=False,
133+
use_reduce=False):
134+
main_prog = fluid.framework.Program()
135+
startup_prog = fluid.framework.Program()
136+
startup_prog.random_seed = 1
137+
with prog_scope_guard(main_prog=main_prog, startup_prog=startup_prog):
138+
139+
data = fluid.layers.data(
140+
name="words", shape=[1], dtype="int64", lod_level=1)
141+
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
142+
143+
avg_cost = model(data, label, len(self.word_dict))
144+
145+
param_list = [(var, var * self.learning_rate)
146+
for var in main_prog.block(0).all_parameters()]
147+
148+
optimizer = fluid.optimizer.Adagrad(
149+
learning_rate=self.learning_rate)
150+
151+
optimizer.minimize(avg_cost)
152+
153+
for params in param_list:
154+
updated_p = fluid.layers.elementwise_sub(
155+
x=params[0], y=params[1])
156+
fluid.layers.assign(input=updated_p, output=params[0])
157+
158+
if use_parallel_exe:
159+
loss = self.run_parallel_exe(
160+
place, [data, label],
161+
loss=avg_cost,
162+
use_cuda=True,
163+
use_reduce=use_reduce)
164+
else:
165+
loss = self.run_executor(place, [data, label], loss=avg_cost)
166+
167+
return loss
168+
169+
def test_weight_decay(self):
170+
model = partial(bow_net, is_sparse=False)
171+
for place in get_places():
172+
loss = self.check_weight_decay(place, model, use_parallel_exe=False)
173+
174+
loss2 = self.check_weight_decay(
175+
place, model, use_parallel_exe=True, use_reduce=False)
176+
177+
for i in range(len(loss)):
178+
assert np.isclose(a=loss[i], b=loss2[i], rtol=5e-5)
179+
180+
loss3 = self.check_weight_decay(
181+
place, model, use_parallel_exe=True, use_reduce=True)
182+
183+
for i in range(len(loss)):
184+
assert np.isclose(a=loss[i], b=loss3[i], rtol=5e-5)
185+
186+
187+
if __name__ == '__main__':
188+
unittest.main()

0 commit comments

Comments
 (0)