|
| 1 | +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +from __future__ import print_function |
| 16 | +import contextlib |
| 17 | + |
| 18 | +import unittest |
| 19 | +from functools import partial |
| 20 | +import numpy as np |
| 21 | +import paddle |
| 22 | +import paddle.fluid.core as core |
| 23 | + |
| 24 | +import paddle.fluid as fluid |
| 25 | + |
| 26 | + |
| 27 | +def get_places(): |
| 28 | + places = [] |
| 29 | + if core.is_compiled_with_cuda(): |
| 30 | + places.append(core.CUDAPlace(0)) |
| 31 | + return places |
| 32 | + |
| 33 | + |
| 34 | +@contextlib.contextmanager |
| 35 | +def prog_scope_guard(main_prog, startup_prog): |
| 36 | + scope = fluid.core.Scope() |
| 37 | + with fluid.unique_name.guard(): |
| 38 | + with fluid.scope_guard(scope): |
| 39 | + with fluid.program_guard(main_prog, startup_prog): |
| 40 | + yield |
| 41 | + |
| 42 | + |
| 43 | +def bow_net(data, |
| 44 | + label, |
| 45 | + dict_dim, |
| 46 | + is_sparse=False, |
| 47 | + emb_dim=128, |
| 48 | + hid_dim=128, |
| 49 | + hid_dim2=96, |
| 50 | + class_dim=2): |
| 51 | + """ |
| 52 | + BOW net |
| 53 | + This model is from https://github.com/PaddlePaddle/models: |
| 54 | + fluid/PaddleNLP/text_classification/nets.py |
| 55 | + """ |
| 56 | + emb = fluid.layers.embedding( |
| 57 | + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim]) |
| 58 | + bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') |
| 59 | + bow_tanh = fluid.layers.tanh(bow) |
| 60 | + fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") |
| 61 | + fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh") |
| 62 | + prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax") |
| 63 | + cost = fluid.layers.cross_entropy(input=prediction, label=label) |
| 64 | + avg_cost = fluid.layers.mean(x=cost) |
| 65 | + |
| 66 | + return avg_cost |
| 67 | + |
| 68 | + |
| 69 | +class TestWeightDecay(unittest.TestCase): |
| 70 | + def setUp(self): |
| 71 | + self.word_dict = paddle.dataset.imdb.word_dict() |
| 72 | + reader = paddle.batch( |
| 73 | + paddle.dataset.imdb.train(self.word_dict), batch_size=4)() |
| 74 | + self.train_data = [next(reader) for _ in range(5)] |
| 75 | + self.learning_rate = .5 |
| 76 | + |
| 77 | + def run_executor(self, place, feed_list, loss): |
| 78 | + exe = fluid.Executor(place) |
| 79 | + feeder = fluid.DataFeeder(feed_list=feed_list, place=place) |
| 80 | + exe.run(fluid.default_startup_program()) |
| 81 | + main_prog = fluid.default_main_program() |
| 82 | + loss_set = [] |
| 83 | + for data in self.train_data: |
| 84 | + out = exe.run(main_prog, |
| 85 | + feed=feeder.feed(data), |
| 86 | + fetch_list=[loss.name]) |
| 87 | + |
| 88 | + print("loss %s" % (np.average(out))) |
| 89 | + loss_set.append(np.average(out)) |
| 90 | + |
| 91 | + return loss_set |
| 92 | + |
| 93 | + def run_parallel_exe(self, |
| 94 | + place, |
| 95 | + feed_list, |
| 96 | + loss, |
| 97 | + use_cuda=True, |
| 98 | + use_reduce=False, |
| 99 | + use_fast_executor=False, |
| 100 | + use_ir_memory_optimize=False): |
| 101 | + exe = fluid.Executor(place) |
| 102 | + feeder = fluid.DataFeeder(feed_list=feed_list, place=place) |
| 103 | + exe.run(fluid.default_startup_program()) |
| 104 | + |
| 105 | + exec_strategy = fluid.ExecutionStrategy() |
| 106 | + if use_fast_executor: |
| 107 | + exec_strategy.use_experimental_executor = True |
| 108 | + |
| 109 | + build_strategy = fluid.BuildStrategy() |
| 110 | + build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce \ |
| 111 | + if use_reduce else fluid.BuildStrategy.ReduceStrategy.AllReduce |
| 112 | + build_strategy.memory_optimize = use_ir_memory_optimize |
| 113 | + |
| 114 | + parallel_exe = fluid.ParallelExecutor( |
| 115 | + use_cuda, |
| 116 | + loss_name=loss.name, |
| 117 | + exec_strategy=exec_strategy, |
| 118 | + build_strategy=build_strategy) |
| 119 | + |
| 120 | + loss_set = [] |
| 121 | + for data in self.train_data: |
| 122 | + out = parallel_exe.run(feed=feeder.feed(data), |
| 123 | + fetch_list=[loss.name]) |
| 124 | + print("loss %s" % (np.average(out))) |
| 125 | + loss_set.append(np.average(out)) |
| 126 | + |
| 127 | + return loss_set |
| 128 | + |
| 129 | + def check_weight_decay(self, |
| 130 | + place, |
| 131 | + model, |
| 132 | + use_parallel_exe=False, |
| 133 | + use_reduce=False): |
| 134 | + main_prog = fluid.framework.Program() |
| 135 | + startup_prog = fluid.framework.Program() |
| 136 | + startup_prog.random_seed = 1 |
| 137 | + with prog_scope_guard(main_prog=main_prog, startup_prog=startup_prog): |
| 138 | + |
| 139 | + data = fluid.layers.data( |
| 140 | + name="words", shape=[1], dtype="int64", lod_level=1) |
| 141 | + label = fluid.layers.data(name="label", shape=[1], dtype="int64") |
| 142 | + |
| 143 | + avg_cost = model(data, label, len(self.word_dict)) |
| 144 | + |
| 145 | + param_list = [(var, var * self.learning_rate) |
| 146 | + for var in main_prog.block(0).all_parameters()] |
| 147 | + |
| 148 | + optimizer = fluid.optimizer.Adagrad( |
| 149 | + learning_rate=self.learning_rate) |
| 150 | + |
| 151 | + optimizer.minimize(avg_cost) |
| 152 | + |
| 153 | + for params in param_list: |
| 154 | + updated_p = fluid.layers.elementwise_sub( |
| 155 | + x=params[0], y=params[1]) |
| 156 | + fluid.layers.assign(input=updated_p, output=params[0]) |
| 157 | + |
| 158 | + if use_parallel_exe: |
| 159 | + loss = self.run_parallel_exe( |
| 160 | + place, [data, label], |
| 161 | + loss=avg_cost, |
| 162 | + use_cuda=True, |
| 163 | + use_reduce=use_reduce) |
| 164 | + else: |
| 165 | + loss = self.run_executor(place, [data, label], loss=avg_cost) |
| 166 | + |
| 167 | + return loss |
| 168 | + |
| 169 | + def test_weight_decay(self): |
| 170 | + model = partial(bow_net, is_sparse=False) |
| 171 | + for place in get_places(): |
| 172 | + loss = self.check_weight_decay(place, model, use_parallel_exe=False) |
| 173 | + |
| 174 | + loss2 = self.check_weight_decay( |
| 175 | + place, model, use_parallel_exe=True, use_reduce=False) |
| 176 | + |
| 177 | + for i in range(len(loss)): |
| 178 | + assert np.isclose(a=loss[i], b=loss2[i], rtol=5e-5) |
| 179 | + |
| 180 | + loss3 = self.check_weight_decay( |
| 181 | + place, model, use_parallel_exe=True, use_reduce=True) |
| 182 | + |
| 183 | + for i in range(len(loss)): |
| 184 | + assert np.isclose(a=loss[i], b=loss3[i], rtol=5e-5) |
| 185 | + |
| 186 | + |
| 187 | +if __name__ == '__main__': |
| 188 | + unittest.main() |
0 commit comments