Skip to content

Commit 828959c

Browse files
chunyuanwchunyuanw
committed
add UT for deconv
1 parent 23438ff commit 828959c

File tree

3 files changed

+253
-109
lines changed

3 files changed

+253
-109
lines changed

‎tests/cpu/test_bf16_lazy_reorder.py‎

Lines changed: 121 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from functools import reduce
1111
import copy
1212
import sys
13+
import itertools
1314
import torch
1415
import intel_pytorch_extension as ipex
1516

@@ -187,76 +188,134 @@ def test_Conv2d_backward(self):
187188
self.assertEqual(in_man_bf16.grad.float(), in_auto_mix.grad.float())
188189

189190
class TestDeconv(TestCase):
190-
def test_Deconv2d_with_cpu(self):
191+
def _deconv_params_list(self):
192+
params_dict = {
193+
"input_height": [8],
194+
"input_width": [8],
195+
"input_depth": [8],
196+
"input_channel_per_group": [10],
197+
"output_channel_per_group": [10],
198+
"kernel_size": [3, 4],
199+
"bias": [False, True],
200+
"stride": [2], # [1, 2]
201+
"padding": [1, 2],
202+
"output_padding": [2],
203+
"groups": [1, 2],
204+
"dilation": [1, 3, 4],
205+
}
206+
207+
params_list = []
208+
209+
for key, value in params_dict.items():
210+
params_list.append(value)
211+
return params_list
212+
213+
def _test_deconv(self, dims):
191214
rand_seed = int(get_rand_seed())
192215
print("{} rand sed: {}".format(sys._getframe().f_code.co_name, rand_seed))
193216
torch.manual_seed(rand_seed)
194217

195-
_deconv = torch.nn.ConvTranspose2d(2, 3, (3, 3))
218+
params_list = self._deconv_params_list()
196219

197-
deconv_man_bf16 = copy.deepcopy(_deconv).to(device=device).to(torch.bfloat16)
198-
deconv_auto_mix = copy.deepcopy(_deconv).to(device=device)
199-
deconv_auto_mix_train =copy.deepcopy(_deconv).to(device=device)
220+
for input_width, input_height, input_depth, input_channel_per_group, output_channel_per_group, kernel_size, bias, stride, padding, output_padding, groups, dilation in itertools.product(*params_list):
221+
if (output_padding < stride or output_padding < dilation) \
222+
and ((input_height - 1) * stride - 2 * padding + dilation * (kernel_size -1 ) + output_padding + 1 > 0) \
223+
and ((input_width - 1) * stride - 2 * padding + dilation * (kernel_size -1 ) + output_padding + 1 > 0) \
224+
and ((input_depth - 1) * stride - 2 * padding + dilation * (kernel_size -1 ) + output_padding + 1 > 0):
200225

201-
_in_cpu = torch.rand((1, 2, 7, 7))
202-
in_auto_mix = _in_cpu.to(device=device)
203-
in_man_bf16 = in_auto_mix.to(torch.bfloat16)
204-
205-
in_auto_mix_train = _in_cpu.to(device=device)
226+
ic = input_channel_per_group * groups
227+
oc = output_channel_per_group * groups
206228

207-
res_cpu_fp32 = _deconv(_in_cpu)
229+
if dims == 2:
230+
module = torch.nn.ConvTranspose2d(ic, oc, kernel_size=kernel_size, stride=stride, padding=padding, output_padding=output_padding, groups=groups, bias=bias, dilation=dilation)
231+
x = torch.rand((2, ic, input_height, input_width))
232+
elif dims == 3:
233+
module = torch.nn.ConvTranspose3d(ic, oc, kernel_size=kernel_size, stride=stride, padding=padding, output_padding=output_padding, groups=groups, bias=bias, dilation=dilation)
234+
x = torch.rand((2, ic, input_depth, input_height, input_width))
208235

209-
with AutoDNNL(True), AutoMixPrecision(False):
210-
res_man_bf16 = deconv_man_bf16(in_man_bf16)
211-
self.assertEqual(res_man_bf16.dtype, torch.bfloat16)
212-
self.assertEqual(res_cpu_fp32.bfloat16().float(), res_man_bf16, 1e-2)
213-
214-
with AutoMixPrecision(True):
215-
self.assertEqual(in_auto_mix.dtype, torch.float)
216-
self.assertFalse(ipex.core.is_bf16_dil_tensor(in_auto_mix))
217-
res_auto_bf16 = deconv_auto_mix(in_auto_mix)
218-
self.assertTrue(ipex.core.is_bf16_dil_tensor(res_auto_bf16))
219-
self.assertEqual(res_man_bf16.float(), res_auto_bf16.float(), 1e-2)
220-
221-
with AutoMixPrecision(True, train=True):
222-
self.assertEqual(in_auto_mix_train.dtype, torch.float)
223-
self.assertFalse(ipex.core.is_bf16_dil_tensor(in_auto_mix_train))
224-
res_auto_bf16_train = deconv_auto_mix_train(in_auto_mix_train)
225-
self.assertTrue(ipex.core.is_bf16_dil_tensor(res_auto_bf16_train))
226-
self.assertEqual(res_man_bf16.float(), res_auto_bf16_train.float(), 1e-2)
227-
228-
def test_Deconv2d_backward(self):
229-
rand_seed = int(get_rand_seed())
230-
print("{} rand sed: {}".format(sys._getframe().f_code.co_name, rand_seed))
231-
torch.manual_seed(rand_seed)
232-
233-
input = torch.rand(2, 10, 8, 8)
234-
for bias in [True, False]:
235-
_deconv = torch.nn.ConvTranspose2d(10, 10,
236-
kernel_size=4, stride=2, bias=bias)
237-
deconv_man_bf16 =copy.deepcopy(_deconv).to(device=device).to(torch.bfloat16)
238-
deconv_auto_mix =copy.deepcopy(_deconv).to(device=device)
239-
_in_cpu = input.clone().requires_grad_()
240-
in_auto_mix = input.clone().to(device=device).requires_grad_()
241-
in_man_bf16 = input.clone().to(device=device).to(torch.bfloat16).requires_grad_()
242-
out_cpu = _deconv(_in_cpu).sum()
243-
out_cpu.backward()
244-
with AutoDNNL(True), AutoMixPrecision(False, train=True):
245-
out_man_bf16 = deconv_man_bf16(in_man_bf16).sum()
246-
out_man_bf16.backward()
247-
self.assertEqual(in_man_bf16.grad.dtype, torch.bfloat16)
248-
self.assertEqual(_in_cpu.grad.bfloat16().float(), in_man_bf16.grad, 1e-2)
249-
250-
with AutoMixPrecision(True, train=True):
251-
self.assertEqual(in_auto_mix.dtype, torch.float)
252-
self.assertFalse(ipex.core.is_bf16_dil_tensor(in_auto_mix))
253-
out_auto_bf16 = deconv_auto_mix(in_auto_mix).sum()
254-
out_auto_bf16.backward()
255-
self.assertTrue(ipex.core.is_bf16_dil_tensor(in_auto_mix.grad))
256-
self.assertFalse(ipex.core.is_bf16_dil_tensor(deconv_auto_mix.weight))
257-
self.assertFalse(ipex.core.is_bf16_dil_tensor(deconv_auto_mix.weight.grad))
258-
self.assertEqual(in_man_bf16.grad.float(), in_auto_mix.grad.float())
236+
module_auto_mix_infer = copy.deepcopy(module).to(device=device)
237+
module_auto_mix_train = copy.deepcopy(module).to(device=device)
259238

239+
x_aten = x.clone().requires_grad_()
240+
x_auto_mix_infer = x.clone().to(device=device).requires_grad_()
241+
x_auto_mix_train = x.clone().to(device=device).requires_grad_()
242+
243+
y_aten = module(x_aten)
244+
y_aten.sum().backward()
245+
246+
with AutoDNNL(True), AutoMixPrecision(True, train=False):
247+
self.assertEqual(x_auto_mix_infer.dtype, torch.float)
248+
self.assertFalse(ipex.core.is_bf16_dil_tensor(x_auto_mix_infer))
249+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_infer.weight))
250+
if bias:
251+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_infer.bias))
252+
253+
y_auto_mix_infer = module_auto_mix_infer(x_auto_mix_infer)
254+
y_auto_mix_infer.sum().backward()
255+
256+
if padding - output_padding + stride > 0:
257+
self.assertTrue(ipex.core.is_bf16_dil_tensor(x_auto_mix_infer.grad))
258+
self.assertTrue(ipex.core.is_bf16_dil_tensor(module_auto_mix_infer.weight))
259+
if bias:
260+
self.assertTrue(ipex.core.is_bf16_dil_tensor(module_auto_mix_infer.bias))
261+
262+
self.assertEqual(
263+
y_aten, y_auto_mix_infer, 1e-2)
264+
265+
# mkldnn does not support the case where:
266+
# padding - output_padding + stride <= 0
267+
# while PyTorch supports this case, will fallback in this case
268+
else:
269+
# threshold because input has been reordered to bf16??
270+
self.assertEqual(
271+
y_aten, y_auto_mix_infer, 4e-3)
272+
273+
with AutoDNNL(True), AutoMixPrecision(True, train=True):
274+
self.assertEqual(x_auto_mix_train.dtype, torch.float)
275+
self.assertFalse(ipex.core.is_bf16_dil_tensor(x_auto_mix_train))
276+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_train.weight))
277+
if bias:
278+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_train.bias))
279+
280+
y_auto_mix_train = module_auto_mix_train(x_auto_mix_train)
281+
y_auto_mix_train.sum().backward()
282+
283+
if padding - output_padding + stride > 0:
284+
self.assertTrue(ipex.core.is_bf16_dil_tensor(x_auto_mix_train.grad))
285+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_train.weight))
286+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_train.weight.grad))
287+
if bias:
288+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_train.bias))
289+
self.assertFalse(ipex.core.is_bf16_dil_tensor(module_auto_mix_train.bias.grad))
290+
291+
self.assertEqual(
292+
y_aten, y_auto_mix_train, 1e-2)
293+
self.assertEqual(
294+
module.weight.grad, module_auto_mix_train.weight.grad, 5e-1)
295+
self.assertEqual(
296+
x_aten.grad, x_auto_mix_train.grad, 1e-2)
297+
if bias:
298+
self.assertEqual(module.bias.grad, module_auto_mix_train.bias.grad)
299+
300+
# mkldnn does not support the case where:
301+
# padding - output_padding + stride <= 0
302+
# while PyTorch supports this case, will fallback in this case
303+
else:
304+
# threshold because input has been reordered to bf16??
305+
self.assertEqual(
306+
y_aten, y_auto_mix_train, 3e-3)
307+
self.assertEqual(
308+
module.weight.grad, module_auto_mix_train.weight.grad, 2e-1)
309+
self.assertEqual(
310+
x_aten.grad, x_auto_mix_train.grad)
311+
if bias:
312+
self.assertEqual(module.bias.grad, module_auto_mix_train.bias.grad)
313+
314+
def test_deconv2d(self):
315+
self._test_deconv(dims=2)
316+
317+
def test_deconv3d(self):
318+
self._test_deconv(dims=3)
260319

261320
class TestBatchNorm(TestCase):
262321
def test_batch_norm2d(self):
@@ -1033,6 +1092,7 @@ def test_linear(self):
10331092

10341093
def test_linear_backward(self):
10351094
rand_seed = int(get_rand_seed())
1095+
# rand_seed = 1600407821102260224 # self.assertEqual(_in_cpu.grad.bfloat16().float(), in_man_bf16.grad, 2e-2) AssertionError: tensor(0.0312) not less than or equal to 0.02
10361096
print("{} rand sed: {}".format(sys._getframe().f_code.co_name, rand_seed))
10371097
torch.manual_seed(rand_seed)
10381098
in_features = torch.randint(3, 10, (1,)).item()

‎tests/cpu/test_lazy_reorder.py‎

Lines changed: 111 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from functools import reduce
1111
import copy
1212
import sys
13+
import itertools
1314
import torch
1415
import intel_pytorch_extension as ipex
1516

@@ -106,27 +107,123 @@ def test_seq_conv(self):
106107
self.assertEqual(res_cpu, res_dpcpp.to('cpu'))
107108

108109
class TestDeconv(TestCase):
109-
def test_Deconv2d_with_cpu(self):
110+
def _deconv_params_list(self):
111+
params_dict = {
112+
"input_height": [8],
113+
"input_width": [8],
114+
"input_depth": [8],
115+
"input_channel_per_group": [10],
116+
"output_channel_per_group": [10],
117+
"kernel_size": [3, 4],
118+
"bias": [False, True],
119+
"stride": [2], # [1, 2]
120+
"padding": [1, 2],
121+
"output_padding": [2],
122+
"groups": [1, 2],
123+
"dilation": [1, 3, 4],
124+
}
125+
126+
params_list = []
127+
128+
for key, value in params_dict.items():
129+
params_list.append(value)
130+
return params_list
131+
132+
def _test_deconv(self, dims):
110133
rand_seed = int(get_rand_seed())
111134
print("{} rand sed: {}".format(sys._getframe().f_code.co_name, rand_seed))
112135
torch.manual_seed(rand_seed)
113-
deconv_cpu = torch.nn.ConvTranspose2d(2, 3, (3, 3))
114-
deconv_dpcpp = torch.nn.ConvTranspose2d(2, 3, (3, 3)).to(device=device)
115136

116-
deconv_dpcpp.weight.data = deconv_cpu.weight.data.to(device=device)
117-
deconv_dpcpp.bias.data = deconv_cpu.bias.data.to(device=device)
137+
params_list = self._deconv_params_list()
138+
139+
for input_width, input_height, input_depth, input_channel_per_group, output_channel_per_group, kernel_size, bias, stride, padding, output_padding, groups, dilation in itertools.product(*params_list):
140+
if (output_padding < stride or output_padding < dilation) \
141+
and ((input_height - 1) * stride - 2 * padding + dilation * (kernel_size -1 ) + output_padding + 1 > 0) \
142+
and ((input_width - 1) * stride - 2 * padding + dilation * (kernel_size -1 ) + output_padding + 1 > 0) \
143+
and ((input_depth - 1) * stride - 2 * padding + dilation * (kernel_size -1 ) + output_padding + 1 > 0):
144+
145+
# mkldnn does not support the case where:
146+
# padding - output_padding + stride <= 0
147+
# while PyTorch supports this case, will fallback in this case
148+
# input_width = 8
149+
# input_height = 8
150+
# input_channel_per_group = 10
151+
# output_channel_per_group = 10
152+
# kernel_size = 4
153+
# bias = False
154+
# stride = 1
155+
# padding = 1
156+
# output_padding = 2
157+
# groups = 1
158+
# dilation = 3
159+
160+
ic = input_channel_per_group * groups
161+
oc = output_channel_per_group * groups
162+
163+
if dims == 2:
164+
module = torch.nn.ConvTranspose2d(ic, oc, kernel_size=kernel_size, stride=stride, padding=padding, output_padding=output_padding, groups=groups, bias=bias, dilation=dilation)
165+
x = torch.rand((2, ic, input_height, input_width))
166+
elif dims == 3:
167+
module = torch.nn.ConvTranspose3d(ic, oc, kernel_size=kernel_size, stride=stride, padding=padding, output_padding=output_padding, groups=groups, bias=bias, dilation=dilation)
168+
x = torch.rand((2, ic, input_depth, input_height, input_width))
169+
170+
module_dpcpp = copy.deepcopy(module).to(device=device)
171+
module_fallback = copy.deepcopy(module).to(device=device)
172+
module_fallback_ = copy.deepcopy(module).to(device=device)
173+
174+
x_aten = x.clone().requires_grad_()
175+
x_dpcpp = x.clone().to(device=device).requires_grad_()
176+
x_fallback = x.clone().to(device=device).requires_grad_()
177+
x_fallback_ = x.clone().to(device=device).requires_grad_()
118178

119-
input_cpu = torch.rand((1, 2, 7, 7))
120-
input_dpcpp = input_cpu.to(device=device)
179+
y_aten = module(x_aten)
180+
y_aten.sum().backward()
121181

122-
ipex.core.enable_auto_dnnl()
123-
out_dpcpp = deconv_dpcpp(input_dpcpp)
182+
# test dnnl
183+
with AutoDNNL(True):
184+
y_dpcpp = module_dpcpp(x_dpcpp)
185+
y_dpcpp.sum().backward()
186+
187+
self.assertEqual(
188+
y_aten, y_dpcpp)
189+
self.assertEqual(
190+
module.weight.grad, module_dpcpp.weight.grad, 1e-3)
191+
self.assertEqual(x_aten.grad, x_dpcpp.grad)
192+
if bias:
193+
self.assertEqual(module.bias.grad, module_dpcpp.bias.grad)
124194

125-
ipex.core.disable_auto_dnnl()
126-
out_dpcpp_cpu = out_dpcpp.to('cpu')
127-
out_cpu = deconv_cpu(input_cpu)
128-
self.assertEqual(out_dpcpp.size(), out_cpu.size())
129-
self.assertEqual(out_cpu, out_dpcpp_cpu)
195+
# test fallback
196+
with AutoDNNL(False):
197+
y_fallback = module_fallback(x_fallback)
198+
y_fallback.sum().backward()
199+
200+
self.assertEqual(
201+
y_aten, y_fallback)
202+
self.assertEqual(
203+
module.weight.grad, module_fallback.weight.grad)
204+
self.assertEqual(x_aten.grad, x_fallback.grad)
205+
if bias:
206+
self.assertEqual(module.bias.grad, module_fallback.bias.grad)
207+
208+
# test fw: dnnl, bw: cpu
209+
with AutoDNNL(True):
210+
y_fallback_ = module_fallback_(x_fallback_)
211+
with AutoDNNL(False):
212+
y_fallback_.sum().backward()
213+
214+
self.assertEqual(
215+
y_aten, y_fallback_)
216+
self.assertEqual(
217+
module.weight.grad, module_fallback_.weight.grad)
218+
self.assertEqual(x_aten.grad, x_fallback_.grad)
219+
if bias:
220+
self.assertEqual(module.bias.grad, module_fallback_.bias.grad)
221+
222+
def test_deconv2d(self):
223+
self._test_deconv(dims=2)
224+
225+
def test_deconv3d(self):
226+
self._test_deconv(dims=3)
130227

131228
def _seq_conf(self, device, rand_seed):
132229
torch.manual_seed(rand_seed)
@@ -150,36 +247,6 @@ def test_seq_deconv(self):
150247
ipex.core.enable_auto_dnnl()
151248
res_dpcpp = self._seq_conf(device, rand_seed)
152249
self.assertEqual(res_cpu, res_dpcpp.to('cpu'))
153-
154-
def test_Deconv2d_backward(self):
155-
rand_seed = int(get_rand_seed())
156-
157-
# rand_seed = 1600072517315694592 # weight.grad AssertionError: tensor(1.9073e-05) not less than or equal to 1e-05
158-
print("{} rand sed: {}".format(sys._getframe().f_code.co_name, rand_seed))
159-
torch.manual_seed(rand_seed)
160-
161-
ipex.core.enable_auto_dnnl()
162-
163-
input = torch.rand(2, 10, 8, 8)
164-
for bias in [False, True]:
165-
input_cpu = input.clone().requires_grad_()
166-
input_dpcpp = input.clone().to(device=device).requires_grad_()
167-
conv_cpu = nn.ConvTranspose2d(10, 10,
168-
kernel_size=4, stride=2, bias=bias)
169-
conv_dpcpp = copy.deepcopy(conv_cpu).to(device=device)
170-
out_cpu = conv_cpu(input_cpu).sum()
171-
out_dpcpp = conv_dpcpp(input_dpcpp).sum()
172-
out_cpu.backward()
173-
out_dpcpp.backward()
174-
175-
self.assertEqual(input_cpu.grad, input_dpcpp.grad)
176-
177-
# TODO threshold
178-
self.assertEqual(conv_cpu.weight.grad, conv_dpcpp.weight.grad, 2e-5)
179-
if bias:
180-
self.assertEqual(conv_cpu.bias.grad, conv_dpcpp.bias.grad)
181-
182-
183250

184251
class TestBinaryOp(TestCase):
185252
def test_add(self):

0 commit comments

Comments
 (0)