Skip to content

Commit f158829

Browse files
committed
test=develop
1 parent 6330c14 commit f158829

File tree

2 files changed

+151
-107
lines changed

2 files changed

+151
-107
lines changed

paddle/fluid/operators/rmsprop_op.h

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -131,21 +131,21 @@ template <typename DeviceContext, typename T>
131131
class RmspropOpKernel : public framework::OpKernel<T> {
132132
public:
133133
void Compute(const framework::ExecutionContext &ctx) const override {
134-
using Tensor = framework::LoDTensor;
134+
using LoDTensor = framework::LoDTensor;
135135
auto *grad_var = ctx.InputVar("Grad");
136-
auto *param_out = ctx.Output<Tensor>("ParamOut");
137-
auto *moment_out = ctx.Output<Tensor>("MomentOut");
138-
auto *mean_square_out = ctx.Output<Tensor>("MeanSquareOut");
136+
auto *param_out = ctx.Output<LoDTensor>("ParamOut");
137+
auto *moment_out = ctx.Output<LoDTensor>("MomentOut");
138+
auto *mean_square_out = ctx.Output<LoDTensor>("MeanSquareOut");
139139

140140
auto epsilon = static_cast<T>(ctx.Attr<float>("epsilon"));
141141
auto rho = static_cast<T>(ctx.Attr<float>("decay"));
142142
auto momentum = static_cast<T>(ctx.Attr<float>("momentum"));
143143
bool centered = ctx.Attr<bool>("centered");
144144

145-
auto &p_tensor = *ctx.Input<Tensor>("Param");
146-
auto &ms_tensor = *ctx.Input<Tensor>("MeanSquare");
147-
auto &lr_tensor = *ctx.Input<Tensor>("LearningRate");
148-
auto &mom_tensor = *ctx.Input<Tensor>("Moment");
145+
auto &p_tensor = *ctx.Input<LoDTensor>("Param");
146+
auto &ms_tensor = *ctx.Input<LoDTensor>("MeanSquare");
147+
auto &lr_tensor = *ctx.Input<LoDTensor>("LearningRate");
148+
auto &mom_tensor = *ctx.Input<LoDTensor>("Moment");
149149

150150
PADDLE_ENFORCE_EQ(&p_tensor, param_out,
151151
"Param and ParamOut must be the same Tensor");
@@ -157,8 +157,8 @@ class RmspropOpKernel : public framework::OpKernel<T> {
157157
auto &dev_ctx = ctx.template device_context<DeviceContext>();
158158
size_t limit = static_cast<size_t>(ms_tensor.numel());
159159

160-
if (grad_var->IsType<Tensor>()) {
161-
auto &grad_tensor = grad_var->Get<Tensor>();
160+
if (grad_var->IsType<LoDTensor>()) {
161+
auto &grad_tensor = grad_var->Get<LoDTensor>();
162162

163163
if (std::is_same<DeviceContext, platform::CPUDeviceContext>::value) {
164164
auto &place =
@@ -176,9 +176,9 @@ class RmspropOpKernel : public framework::OpKernel<T> {
176176

177177
ms_out.device(place) = rho * ms + (1 - rho) * g * g;
178178
if (centered) {
179-
auto &mg_tensor = *ctx.Input<Tensor>("MeanGrad");
179+
auto &mg_tensor = *ctx.Input<LoDTensor>("MeanGrad");
180180
auto mg = EigenVector<T>::Flatten(mg_tensor);
181-
auto *mean_grad_out = ctx.Output<Tensor>("MeanGradOut");
181+
auto *mean_grad_out = ctx.Output<LoDTensor>("MeanGradOut");
182182
PADDLE_ENFORCE(&mg_tensor, mean_grad_out,
183183
"MeanGrad and MeanGradOut must be the same Tensor");
184184
auto mg_out = EigenVector<T>::Flatten(*mean_grad_out);
@@ -196,8 +196,8 @@ class RmspropOpKernel : public framework::OpKernel<T> {
196196
DenseRmspropGradFunctor<T> grad_func(grad_tensor.data<T>());
197197
platform::ForRange<DeviceContext> for_range(dev_ctx, limit);
198198
if (centered) {
199-
auto &mg_tensor = *ctx.Input<Tensor>("MeanGrad");
200-
auto *mean_grad_out = ctx.Output<Tensor>("MeanGradOut");
199+
auto &mg_tensor = *ctx.Input<LoDTensor>("MeanGrad");
200+
auto *mean_grad_out = ctx.Output<LoDTensor>("MeanGradOut");
201201
PADDLE_ENFORCE(&mg_tensor, mean_grad_out,
202202
"MeanGrad and MeanGradOut must be the same Tensor");
203203
for_range(CenteredRmspropFunctor<T, DenseRmspropGradFunctor<T>>(
@@ -241,8 +241,8 @@ class RmspropOpKernel : public framework::OpKernel<T> {
241241
row_numel, row_count);
242242

243243
if (centered) {
244-
auto &mg_tensor = *ctx.Input<Tensor>("MeanGrad");
245-
auto *mean_grad_out = ctx.Output<Tensor>("MeanGradOut");
244+
auto &mg_tensor = *ctx.Input<LoDTensor>("MeanGrad");
245+
auto *mean_grad_out = ctx.Output<LoDTensor>("MeanGradOut");
246246
PADDLE_ENFORCE(&mg_tensor, mean_grad_out,
247247
"MeanGrad and MeanGradOut must be the same Tensor");
248248
for_range(CenteredRmspropFunctor<T, SparseRmspropGradFunctor<T>>(

python/paddle/fluid/tests/unittests/test_rmsprop_op.py

Lines changed: 135 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -19,29 +19,72 @@
1919
import numpy as np
2020
import paddle.fluid.core as core
2121
from paddle.fluid.op import Operator
22+
import paddle.fluid as fluid
23+
24+
25+
def create_selected_rows_and_tensor(scope, place, height, row_num,
26+
embedding_size):
27+
sr = scope.var("@selected_rows@").get_selected_rows()
28+
tensor = scope.var("grad").get_tensor()
29+
30+
rows = np.random.random_integers(
31+
low=0, high=height - 1, size=[row_num, ]).astype('int64')
32+
sr_val = np.random.random(size=[row_num, embedding_size]).astype('float32')
33+
34+
sr.set_height(height)
35+
sr.set_rows(rows)
36+
sr.get_tensor().set(sr_val, place)
37+
38+
tensor_val = np.zeros(shape=[height, embedding_size], dtype='float32')
39+
for i in range(row_num):
40+
row = rows[i]
41+
tensor_val[row, :] = tensor_val[row, :] + sr_val[i, :]
42+
43+
tensor.set(tensor_val, place)
44+
return tensor_val, sr_val
2245

2346

2447
class TestBase(unittest.TestCase):
25-
def setup(self, centered, epsilon=1e-6):
48+
def setup(self,
49+
place,
50+
is_sparse,
51+
centered,
52+
size,
53+
row_num=None,
54+
epsilon=1e-6):
2655
np.random.seed(5) # fix seed
2756

57+
self.scope = fluid.global_scope()
58+
self.place = place
59+
2860
self.param_name = "param"
29-
self.param = np.random.random((123, 321)).astype("float32")
61+
self.param = np.random.random(size).astype("float32")
3062

3163
self.mean_square_name = "mean_square"
32-
self.mean_square = np.random.random((123, 321)).astype("float32")
64+
self.mean_square = np.random.uniform(
65+
low=1, high=2, size=size).astype("float32")
3366

3467
self.mean_grad_name = "mean_grad"
35-
self.mean_grad = np.random.random((123, 321)).astype("float32")
68+
self.mean_grad = np.random.random(size).astype("float32")
3669

3770
self.lr_name = "lr"
3871
self.learning_rate = np.array([0.01]).astype("float32")
3972

4073
self.grad_name = "grad"
41-
self.grad = np.random.random((123, 321)).astype("float32")
74+
75+
self.is_sparse = is_sparse
76+
if self.is_sparse:
77+
self.grad_sr_name = "@selected_rows@"
78+
self.grad, self.grad_sr = create_selected_rows_and_tensor(
79+
self.scope, place, size[0], row_num, size[1])
80+
else:
81+
self.grad = np.random.random(size).astype("float32")
82+
grad_tensor = self.scope.var(self.grad_name).get_tensor()
83+
grad_tensor.set(self.grad, place)
4284

4385
self.moment_name = "moment"
44-
self.moment = np.zeros((123, 321)).astype("float32")
86+
self.moment = np.random.uniform(
87+
low=0, high=1, size=size).astype("float32")
4588

4689
self.epsilon = epsilon
4790
self.decay = 0.9
@@ -61,118 +104,119 @@ def setup(self, centered, epsilon=1e-6):
61104

62105
self.param_out = self.param - self.moment_out
63106

64-
def check(self,
65-
actual_t,
66-
expect_t,
67-
place,
68-
out_name,
69-
atol=1e-5,
70-
equal_nan=False):
71-
self.assertTrue(
72-
np.allclose(
73-
actual_t, expect_t, atol=atol, equal_nan=equal_nan),
74-
"Output (" + out_name + ") has diff at " + str(place) + "\nExpect "
75-
+ str(expect_t) + "\n" + "But Got" + str(actual_t))
76-
77-
78-
class TestRmspropOp(TestBase):
79-
def check_with_place(self, place, centered, epsilon):
80-
self.setup(centered, epsilon)
81-
scope = core.Scope()
82-
83107
# create and initialize Param Variable
84-
param = scope.var(self.param_name).get_tensor()
85-
param.set(self.param, place)
108+
self.param_tensor = self.scope.var(self.param_name).get_tensor()
109+
self.param_tensor.set(self.param, place)
86110

87-
mean_square = scope.var(self.mean_square_name).get_tensor()
88-
mean_square.set(self.mean_square, place)
111+
self.mean_square_tensor = self.scope.var(
112+
self.mean_square_name).get_tensor()
113+
self.mean_square_tensor.set(self.mean_square, place)
89114

90-
lr = scope.var(self.lr_name).get_tensor()
115+
lr = self.scope.var(self.lr_name).get_tensor()
91116
lr.set(self.learning_rate, place)
92117

93-
grad = scope.var(self.grad_name).get_tensor()
94-
grad.set(self.grad, place)
118+
self.moment_tensor = self.scope.var(self.moment_name).get_tensor()
119+
self.moment_tensor.set(self.moment, place)
95120

96-
moment = scope.var(self.moment_name).get_tensor()
97-
moment.set(self.moment, place)
121+
if self.centered:
122+
self.mean_grad_tensor = self.scope.var(
123+
self.mean_grad_name).get_tensor()
124+
self.mean_grad_tensor.set(self.mean_grad, place)
98125

99-
# create and run sgd operator
126+
def check(self, actual_t, expect_t, place, out_name, atol=1e-5):
127+
self.assertTrue(
128+
np.allclose(
129+
actual_t, expect_t, atol=atol),
130+
"Output (" + out_name + ") has diff at " + str(place) + "\nExpect "
131+
+ str(expect_t) + "\n" + "But Got" + str(actual_t))
100132

101-
if self.centered:
102-
mean_grad = scope.var(self.mean_grad_name).get_tensor()
103-
mean_grad.set(self.mean_grad, place)
104-
105-
rmsprop_op = Operator(
106-
"rmsprop",
107-
Param=self.param_name,
108-
Grad=self.grad_name,
109-
MeanSquare=self.mean_square_name,
110-
MeanGrad=self.mean_grad_name,
111-
Moment=self.moment_name,
112-
LearningRate=self.lr_name,
113-
ParamOut=self.param_name,
114-
MeanSquareOut=self.mean_square_name,
115-
MomentOut=self.moment_name,
116-
MeanGradOut=self.mean_grad_name,
117-
epsilon=self.epsilon,
118-
decay=self.decay,
119-
momentum=self.momentum,
120-
centered=True)
121-
else:
122-
rmsprop_op = Operator(
123-
"rmsprop",
124-
Param=self.param_name,
125-
Grad=self.grad_name,
126-
MeanSquare=self.mean_square_name,
127-
Moment=self.moment_name,
128-
LearningRate=self.lr_name,
129-
ParamOut=self.param_name,
130-
MeanSquareOut=self.mean_square_name,
131-
MomentOut=self.moment_name,
132-
epsilon=self.epsilon,
133-
decay=self.decay,
134-
momentum=self.momentum,
135-
centered=False)
136-
137-
rmsprop_op.run(scope, place)
138-
139-
atol = 1e-5
140-
equal_nan = False
133+
134+
class TestRmspropOp(TestBase):
135+
def check_with_place(self,
136+
place,
137+
is_sparse,
138+
centered,
139+
size,
140+
row_num=None,
141+
epsilon=1e-6):
142+
self.setup(place, is_sparse, centered, size, row_num, epsilon)
143+
self.run_and_check()
144+
145+
def run_and_check(self):
146+
grad_name = self.grad_sr_name if self.is_sparse else self.grad_name
147+
148+
kwargs = {
149+
'Param': self.param_name,
150+
'Grad': grad_name,
151+
'MeanSquare': self.mean_square_name,
152+
'Moment': self.moment_name,
153+
'LearningRate': self.lr_name,
154+
'ParamOut': self.param_name,
155+
'MeanSquareOut': self.mean_square_name,
156+
'MomentOut': self.moment_name,
157+
'epsilon': self.epsilon,
158+
'decay': self.decay,
159+
'momentum': self.momentum,
160+
'centered': self.centered
161+
}
141162

142163
if self.centered:
143-
atol = 1e-3
144-
equal_nan = True
164+
kwargs['MeanGrad'] = self.mean_grad_name
165+
kwargs['MeanGradOut'] = self.mean_grad_name
166+
167+
rmsprop_op = Operator('rmsprop', **kwargs)
168+
atol = 1e-6
169+
170+
rmsprop_op.run(self.scope, self.place)
145171

146172
self.check(
147-
np.array(mean_square), self.ms_out, place, self.mean_square_name)
173+
np.array(self.mean_square_tensor), self.ms_out, self.place,
174+
self.mean_square_name)
148175
self.check(
149-
np.array(moment),
176+
np.array(self.moment_tensor),
150177
self.moment_out,
151-
place,
178+
self.place,
152179
self.moment_name,
153-
atol=atol,
154-
equal_nan=equal_nan)
180+
atol=atol)
155181
self.check(
156-
np.array(param),
182+
np.array(self.param_tensor),
157183
self.param_out,
158-
place,
184+
self.place,
159185
self.param_name,
160-
atol=atol,
161-
equal_nan=equal_nan)
186+
atol=atol)
162187

163188
if self.centered:
164189
self.check(
165-
np.array(mean_grad), self.mg_out, place, self.mean_grad_name)
190+
np.array(self.mean_grad_tensor), self.mg_out, self.place,
191+
self.mean_grad_name)
166192

167193
def test_rmsprop(self):
168194
places = [core.CPUPlace()]
169195
if core.is_compiled_with_cuda():
170196
places.append(core.CUDAPlace(0))
197+
198+
size = (128, 320)
171199
for place in places:
172-
self.check_with_place(place, False, 1e-6)
173-
self.check_with_place(place, False, 1e-10)
174-
self.check_with_place(place, True, 1e-6)
175-
self.check_with_place(place, True, 1e-10)
200+
for centered in [False, True]:
201+
with fluid.scope_guard(core.Scope()):
202+
self.check_with_place(
203+
place, is_sparse=False, centered=centered, size=size)
204+
205+
with fluid.scope_guard(core.Scope()):
206+
self.check_with_place(
207+
place,
208+
is_sparse=True,
209+
centered=centered,
210+
row_num=512,
211+
size=size)
212+
213+
with fluid.scope_guard(core.Scope()):
214+
self.check_with_place(
215+
place,
216+
is_sparse=True,
217+
centered=centered,
218+
row_num=60,
219+
size=size)
176220

177221

178222
if __name__ == "__main__":

0 commit comments

Comments
 (0)