19
19
import numpy as np
20
20
import paddle .fluid .core as core
21
21
from paddle .fluid .op import Operator
22
+ import paddle .fluid as fluid
23
+
24
+
25
+ def create_selected_rows_and_tensor (scope , place , height , row_num ,
26
+ embedding_size ):
27
+ sr = scope .var ("@selected_rows@" ).get_selected_rows ()
28
+ tensor = scope .var ("grad" ).get_tensor ()
29
+
30
+ rows = np .random .random_integers (
31
+ low = 0 , high = height - 1 , size = [row_num , ]).astype ('int64' )
32
+ sr_val = np .random .random (size = [row_num , embedding_size ]).astype ('float32' )
33
+
34
+ sr .set_height (height )
35
+ sr .set_rows (rows )
36
+ sr .get_tensor ().set (sr_val , place )
37
+
38
+ tensor_val = np .zeros (shape = [height , embedding_size ], dtype = 'float32' )
39
+ for i in range (row_num ):
40
+ row = rows [i ]
41
+ tensor_val [row , :] = tensor_val [row , :] + sr_val [i , :]
42
+
43
+ tensor .set (tensor_val , place )
44
+ return tensor_val , sr_val
22
45
23
46
24
47
class TestBase (unittest .TestCase ):
25
- def setup (self , centered , epsilon = 1e-6 ):
48
+ def setup (self ,
49
+ place ,
50
+ is_sparse ,
51
+ centered ,
52
+ size ,
53
+ row_num = None ,
54
+ epsilon = 1e-6 ):
26
55
np .random .seed (5 ) # fix seed
27
56
57
+ self .scope = fluid .global_scope ()
58
+ self .place = place
59
+
28
60
self .param_name = "param"
29
- self .param = np .random .random (( 123 , 321 ) ).astype ("float32" )
61
+ self .param = np .random .random (size ).astype ("float32" )
30
62
31
63
self .mean_square_name = "mean_square"
32
- self .mean_square = np .random .random ((123 , 321 )).astype ("float32" )
64
+ self .mean_square = np .random .uniform (
65
+ low = 1 , high = 2 , size = size ).astype ("float32" )
33
66
34
67
self .mean_grad_name = "mean_grad"
35
- self .mean_grad = np .random .random (( 123 , 321 ) ).astype ("float32" )
68
+ self .mean_grad = np .random .random (size ).astype ("float32" )
36
69
37
70
self .lr_name = "lr"
38
71
self .learning_rate = np .array ([0.01 ]).astype ("float32" )
39
72
40
73
self .grad_name = "grad"
41
- self .grad = np .random .random ((123 , 321 )).astype ("float32" )
74
+
75
+ self .is_sparse = is_sparse
76
+ if self .is_sparse :
77
+ self .grad_sr_name = "@selected_rows@"
78
+ self .grad , self .grad_sr = create_selected_rows_and_tensor (
79
+ self .scope , place , size [0 ], row_num , size [1 ])
80
+ else :
81
+ self .grad = np .random .random (size ).astype ("float32" )
82
+ grad_tensor = self .scope .var (self .grad_name ).get_tensor ()
83
+ grad_tensor .set (self .grad , place )
42
84
43
85
self .moment_name = "moment"
44
- self .moment = np .zeros ((123 , 321 )).astype ("float32" )
86
+ self .moment = np .random .uniform (
87
+ low = 0 , high = 1 , size = size ).astype ("float32" )
45
88
46
89
self .epsilon = epsilon
47
90
self .decay = 0.9
@@ -61,118 +104,119 @@ def setup(self, centered, epsilon=1e-6):
61
104
62
105
self .param_out = self .param - self .moment_out
63
106
64
- def check (self ,
65
- actual_t ,
66
- expect_t ,
67
- place ,
68
- out_name ,
69
- atol = 1e-5 ,
70
- equal_nan = False ):
71
- self .assertTrue (
72
- np .allclose (
73
- actual_t , expect_t , atol = atol , equal_nan = equal_nan ),
74
- "Output (" + out_name + ") has diff at " + str (place ) + "\n Expect "
75
- + str (expect_t ) + "\n " + "But Got" + str (actual_t ))
76
-
77
-
78
- class TestRmspropOp (TestBase ):
79
- def check_with_place (self , place , centered , epsilon ):
80
- self .setup (centered , epsilon )
81
- scope = core .Scope ()
82
-
83
107
# create and initialize Param Variable
84
- param = scope .var (self .param_name ).get_tensor ()
85
- param .set (self .param , place )
108
+ self . param_tensor = self . scope .var (self .param_name ).get_tensor ()
109
+ self . param_tensor .set (self .param , place )
86
110
87
- mean_square = scope .var (self .mean_square_name ).get_tensor ()
88
- mean_square .set (self .mean_square , place )
111
+ self .mean_square_tensor = self .scope .var (
112
+ self .mean_square_name ).get_tensor ()
113
+ self .mean_square_tensor .set (self .mean_square , place )
89
114
90
- lr = scope .var (self .lr_name ).get_tensor ()
115
+ lr = self . scope .var (self .lr_name ).get_tensor ()
91
116
lr .set (self .learning_rate , place )
92
117
93
- grad = scope .var (self .grad_name ).get_tensor ()
94
- grad . set (self .grad , place )
118
+ self . moment_tensor = self . scope .var (self .moment_name ).get_tensor ()
119
+ self . moment_tensor . set (self .moment , place )
95
120
96
- moment = scope .var (self .moment_name ).get_tensor ()
97
- moment .set (self .moment , place )
121
+ if self .centered :
122
+ self .mean_grad_tensor = self .scope .var (
123
+ self .mean_grad_name ).get_tensor ()
124
+ self .mean_grad_tensor .set (self .mean_grad , place )
98
125
99
- # create and run sgd operator
126
+ def check (self , actual_t , expect_t , place , out_name , atol = 1e-5 ):
127
+ self .assertTrue (
128
+ np .allclose (
129
+ actual_t , expect_t , atol = atol ),
130
+ "Output (" + out_name + ") has diff at " + str (place ) + "\n Expect "
131
+ + str (expect_t ) + "\n " + "But Got" + str (actual_t ))
100
132
101
- if self .centered :
102
- mean_grad = scope .var (self .mean_grad_name ).get_tensor ()
103
- mean_grad .set (self .mean_grad , place )
104
-
105
- rmsprop_op = Operator (
106
- "rmsprop" ,
107
- Param = self .param_name ,
108
- Grad = self .grad_name ,
109
- MeanSquare = self .mean_square_name ,
110
- MeanGrad = self .mean_grad_name ,
111
- Moment = self .moment_name ,
112
- LearningRate = self .lr_name ,
113
- ParamOut = self .param_name ,
114
- MeanSquareOut = self .mean_square_name ,
115
- MomentOut = self .moment_name ,
116
- MeanGradOut = self .mean_grad_name ,
117
- epsilon = self .epsilon ,
118
- decay = self .decay ,
119
- momentum = self .momentum ,
120
- centered = True )
121
- else :
122
- rmsprop_op = Operator (
123
- "rmsprop" ,
124
- Param = self .param_name ,
125
- Grad = self .grad_name ,
126
- MeanSquare = self .mean_square_name ,
127
- Moment = self .moment_name ,
128
- LearningRate = self .lr_name ,
129
- ParamOut = self .param_name ,
130
- MeanSquareOut = self .mean_square_name ,
131
- MomentOut = self .moment_name ,
132
- epsilon = self .epsilon ,
133
- decay = self .decay ,
134
- momentum = self .momentum ,
135
- centered = False )
136
-
137
- rmsprop_op .run (scope , place )
138
-
139
- atol = 1e-5
140
- equal_nan = False
133
+
134
+ class TestRmspropOp (TestBase ):
135
+ def check_with_place (self ,
136
+ place ,
137
+ is_sparse ,
138
+ centered ,
139
+ size ,
140
+ row_num = None ,
141
+ epsilon = 1e-6 ):
142
+ self .setup (place , is_sparse , centered , size , row_num , epsilon )
143
+ self .run_and_check ()
144
+
145
+ def run_and_check (self ):
146
+ grad_name = self .grad_sr_name if self .is_sparse else self .grad_name
147
+
148
+ kwargs = {
149
+ 'Param' : self .param_name ,
150
+ 'Grad' : grad_name ,
151
+ 'MeanSquare' : self .mean_square_name ,
152
+ 'Moment' : self .moment_name ,
153
+ 'LearningRate' : self .lr_name ,
154
+ 'ParamOut' : self .param_name ,
155
+ 'MeanSquareOut' : self .mean_square_name ,
156
+ 'MomentOut' : self .moment_name ,
157
+ 'epsilon' : self .epsilon ,
158
+ 'decay' : self .decay ,
159
+ 'momentum' : self .momentum ,
160
+ 'centered' : self .centered
161
+ }
141
162
142
163
if self .centered :
143
- atol = 1e-3
144
- equal_nan = True
164
+ kwargs ['MeanGrad' ] = self .mean_grad_name
165
+ kwargs ['MeanGradOut' ] = self .mean_grad_name
166
+
167
+ rmsprop_op = Operator ('rmsprop' , ** kwargs )
168
+ atol = 1e-6
169
+
170
+ rmsprop_op .run (self .scope , self .place )
145
171
146
172
self .check (
147
- np .array (mean_square ), self .ms_out , place , self .mean_square_name )
173
+ np .array (self .mean_square_tensor ), self .ms_out , self .place ,
174
+ self .mean_square_name )
148
175
self .check (
149
- np .array (moment ),
176
+ np .array (self . moment_tensor ),
150
177
self .moment_out ,
151
- place ,
178
+ self . place ,
152
179
self .moment_name ,
153
- atol = atol ,
154
- equal_nan = equal_nan )
180
+ atol = atol )
155
181
self .check (
156
- np .array (param ),
182
+ np .array (self . param_tensor ),
157
183
self .param_out ,
158
- place ,
184
+ self . place ,
159
185
self .param_name ,
160
- atol = atol ,
161
- equal_nan = equal_nan )
186
+ atol = atol )
162
187
163
188
if self .centered :
164
189
self .check (
165
- np .array (mean_grad ), self .mg_out , place , self .mean_grad_name )
190
+ np .array (self .mean_grad_tensor ), self .mg_out , self .place ,
191
+ self .mean_grad_name )
166
192
167
193
def test_rmsprop (self ):
168
194
places = [core .CPUPlace ()]
169
195
if core .is_compiled_with_cuda ():
170
196
places .append (core .CUDAPlace (0 ))
197
+
198
+ size = (128 , 320 )
171
199
for place in places :
172
- self .check_with_place (place , False , 1e-6 )
173
- self .check_with_place (place , False , 1e-10 )
174
- self .check_with_place (place , True , 1e-6 )
175
- self .check_with_place (place , True , 1e-10 )
200
+ for centered in [False , True ]:
201
+ with fluid .scope_guard (core .Scope ()):
202
+ self .check_with_place (
203
+ place , is_sparse = False , centered = centered , size = size )
204
+
205
+ with fluid .scope_guard (core .Scope ()):
206
+ self .check_with_place (
207
+ place ,
208
+ is_sparse = True ,
209
+ centered = centered ,
210
+ row_num = 512 ,
211
+ size = size )
212
+
213
+ with fluid .scope_guard (core .Scope ()):
214
+ self .check_with_place (
215
+ place ,
216
+ is_sparse = True ,
217
+ centered = centered ,
218
+ row_num = 60 ,
219
+ size = size )
176
220
177
221
178
222
if __name__ == "__main__" :
0 commit comments