Skip to content

Commit cd8329f

Browse files
jmitrevsjmitrevs
authored
Merge pull request fastmachinelearning#861 from vloncar/qsep_conv
Support for quantized SeparableConv1D/2D
2 parents 5aacd54 + fc1b878 commit cd8329f

File tree

9 files changed

+185
-42
lines changed

9 files changed

+185
-42
lines changed

‎hls4ml/backends/vivado/passes/convolution_templates.py‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -244,10 +244,12 @@ def __init__(self):
244244
}};\n"""
245245

246246
sepconv1d_function_template = (
247-
'nnet::separable_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {d}, {p}, {z}, {b});'
247+
'nnet::separable_conv_1d_{data_format}<{input_t}, {dw_output_t}, {output_t}, {config}>('
248+
'{input}, {output}, {d}, {p}, {z}, {b});'
248249
)
249250
sepconv2d_function_template = (
250-
'nnet::separable_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {d}, {p}, {z}, {b});'
251+
'nnet::separable_conv_2d_{data_format}<{input_t}, {dw_output_t}, {output_t}, {config}>('
252+
'{input}, {output}, {d}, {p}, {z}, {b});'
251253
)
252254

253255
sepconv1d_include_list = ['nnet_utils/nnet_conv1d.h', 'nnet_utils/nnet_sepconv1d_stream.h']
@@ -273,14 +275,17 @@ def format(self, node):
273275

274276
# Depthwise config
275277
params = self._default_config_params(node)
278+
# Override bias and bias_t since these are zeros in depthwise step of SepConv1D
279+
params['bias'] = params['zero_bias']
280+
params['bias_t'] = params['zero_bias_t']
276281
params['n_filt'] = params['n_chan'] # In depthwise step n_chan == n_filt
277282
params['dilation'] = node.get_attr('dilation', 1)
278283
params['nzeros'] = node.get_weights('depthwise').nzeros
279284
params['index'] = str(node.index) + '_depthwise'
280285
params['weight_t'] = node.get_weights('depthwise').type
281286
params['fill_fn'] = 'FillConv1DBuffer'
282287

283-
if node.get_attr("unscaled"):
288+
if node.get_attr('unscaled'):
284289
params['scale_index_type'] = 'scale_index_unscaled'
285290
else:
286291
params['scale_index_type'] = 'scale_index_regular'
@@ -301,14 +306,11 @@ def format(self, node):
301306
depthwise_mult_config = self.depthwise_mult_template.format(**mult_params)
302307

303308
# Pointwise config
304-
params = self._default_config_params()
305-
input_shape = self.get_input_variable().shape
306-
if self.get_attr('data_format') == 'channels_last':
307-
params['in_width'] = '*'.join([str(k) for k in input_shape[:-1]])
308-
params['n_chan'] = input_shape[-1]
309+
params = self._default_config_params(node)
310+
if node.get_attr('data_format') == 'channels_last':
311+
params['in_width'] = node.get_output_variable().shape[0]
309312
else:
310-
params['in_width'] = '*'.join([str(k) for k in input_shape[1:]])
311-
params['n_chan'] = input_shape[0]
313+
params['in_width'] = node.get_output_variable().shape[1]
312314

313315
params['filt_width'] = 1
314316
params['stride_width'] = 1
@@ -320,7 +322,7 @@ def format(self, node):
320322
params['instructions'] = '0'
321323
params['fill_fn'] = 'FillConv1DBuffer'
322324

323-
if node.get_attr("unscaled"):
325+
if node.get_attr('unscaled'):
324326
params['scale_index_type'] = 'scale_index_unscaled'
325327
else:
326328
params['scale_index_type'] = 'scale_index_regular'
@@ -360,6 +362,7 @@ def __init__(self):
360362

361363
def format(self, node):
362364
params = self._default_function_params(node)
365+
params['dw_output_t'] = node.get_attr('dw_output_t').name
363366
params['data_format'] = 'cf' if node.get_attr('data_format') == 'channels_first' else 'cl'
364367
params['d'] = node.get_weights('depthwise').name
365368
params['p'] = node.get_weights('pointwise').name
@@ -398,12 +401,12 @@ def format(self, node):
398401
params['weight_t'] = node.get_weights('depthwise').type
399402
params['fill_fn'] = 'FillConv2DBuffer'
400403

401-
if node.get_attr("unscaled_h"):
404+
if node.get_attr('unscaled_h'):
402405
params['scale_index_height_type'] = 'scale_index_unscaled'
403406
else:
404407
params['scale_index_height_type'] = 'scale_index_regular'
405408

406-
if node.get_attr("unscaled_w"):
409+
if node.get_attr('unscaled_w'):
407410
params['scale_index_width_type'] = 'scale_index_unscaled'
408411
else:
409412
params['scale_index_width_type'] = 'scale_index_regular'
@@ -443,12 +446,12 @@ def format(self, node):
443446
params['instructions'] = '0'
444447
params['fill_fn'] = 'FillConv2DBuffer'
445448

446-
if node.get_attr("unscaled_h"):
449+
if node.get_attr('unscaled_h'):
447450
params['scale_index_height_type'] = 'scale_index_unscaled'
448451
else:
449452
params['scale_index_height_type'] = 'scale_index_regular'
450453

451-
if node.get_attr("unscaled_w"):
454+
if node.get_attr('unscaled_w'):
452455
params['scale_index_width_type'] = 'scale_index_unscaled'
453456
else:
454457
params['scale_index_width_type'] = 'scale_index_regular'
@@ -487,6 +490,7 @@ def __init__(self):
487490

488491
def format(self, node):
489492
params = self._default_function_params(node)
493+
params['dw_output_t'] = node.get_attr('dw_output_t').name
490494
params['data_format'] = 'cf' if node.get_attr('data_format') == 'channels_first' else 'cl'
491495
params['d'] = node.get_weights('depthwise').name
492496
params['p'] = node.get_weights('pointwise').name

‎hls4ml/backends/vivado/vivado_backend.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
Softmax,
2929
)
3030
from hls4ml.model.optimizer import get_backend_passes, layer_optimizer
31-
from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType
31+
from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType, PackedType
3232
from hls4ml.report import parse_vivado_report
3333
from hls4ml.utils.fixed_point_utils import ceil_log2
3434

@@ -75,6 +75,12 @@ def _register_layer_attributes(self):
7575
attrs.append(ChoiceAttribute('conv_implementation', choices=['LineBuffer', 'Encoded'], default='LineBuffer'))
7676
self.attribute_map[layer] = attrs
7777

78+
sep_conv_layers = [SeparableConv1D, SeparableConv2D]
79+
for layer in sep_conv_layers:
80+
attrs = self.attribute_map.get(layer, [])
81+
attrs.append(TypeAttribute('dw_output', default=FixedPrecisionType(18, 8)))
82+
self.attribute_map[layer] = attrs
83+
7884
def _register_flows(self):
7985
initializers = self._get_layer_initializers()
8086
init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
@@ -288,6 +294,15 @@ def init_sepconv1d(self, layer):
288294
) # TODO Once we have SeparableConv implementation for io_parallel this should be set properly
289295
layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
290296

297+
# Set the output type of the depthwise phase
298+
dw_out_precision, _ = layer.model.config.get_precision(layer, 'dw_output')
299+
dw_out_name = layer.name + '_dw_out_t'
300+
if layer.model.config.get_config_value('IOType') == 'io_stream':
301+
dw_output_t = PackedType(dw_out_name, dw_out_precision, layer.get_attr('n_chan'), n_pack=1)
302+
else:
303+
dw_output_t = NamedType(dw_out_name, dw_out_precision)
304+
layer.set_attr('dw_output_t', dw_output_t)
305+
291306
@layer_optimizer(Conv2D)
292307
def init_conv2d(self, layer):
293308
if len(layer.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
@@ -334,6 +349,15 @@ def init_sepconv2d(self, layer):
334349
) # TODO Once we have SeparableConv implementation for io_parallel this should be set properly
335350
layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
336351

352+
# Set the output type of the depthwise phase
353+
dw_out_precision, _ = layer.model.config.get_precision(layer, 'dw_output')
354+
dw_out_name = layer.name + '_dw_out_t'
355+
if layer.model.config.get_config_value('IOType') == 'io_stream':
356+
dw_output_t = PackedType(dw_out_name, dw_out_precision, layer.get_attr('n_chan'), n_pack=1)
357+
else:
358+
dw_output_t = NamedType(dw_out_name, dw_out_precision)
359+
layer.set_attr('dw_output_t', dw_output_t)
360+
337361
@layer_optimizer(DepthwiseConv2D)
338362
def init_depconv2d(self, layer):
339363
if layer.model.config.is_resource_strategy(layer):

‎hls4ml/converters/keras/qkeras.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,27 @@ def parse_qdepthwiseqconv_layer(keras_layer, input_names, input_shapes, data_rea
5454
layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
5555

5656
layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
57+
58+
if keras_layer['config']['bias_quantizer'] is not None:
59+
layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
60+
else:
61+
layer['bias_quantizer'] = None
62+
63+
return layer, output_shape
64+
65+
66+
@keras_handler('QSeparableConv1D', 'QSeparableConv2D')
67+
def parse_qsepconv_layer(keras_layer, input_names, input_shapes, data_reader):
68+
assert 'QSeparableConv' in keras_layer['class_name']
69+
70+
if '1D' in keras_layer['class_name']:
71+
layer, output_shape = parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader)
72+
elif '2D' in keras_layer['class_name']:
73+
layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
74+
75+
layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
76+
layer['pointwise_quantizer'] = get_quantizer_from_config(keras_layer, 'pointwise')
77+
5778
if keras_layer['config']['bias_quantizer'] is not None:
5879
layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
5980
else:

‎hls4ml/model/layers.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ class Layer:
5656
ConfigurableAttribute('trace', default=False),
5757
TypeAttribute('result'),
5858
]
59-
""""""
6059

6160
@classproperty
6261
def expected_attributes(cls):
@@ -1343,8 +1342,10 @@ def initialize(self):
13431342
'QConv2D': Conv2D,
13441343
'QConv2DBatchnorm': Conv2DBatchnorm,
13451344
'SeparableConv1D': SeparableConv1D,
1345+
'QSeparableConv1D': SeparableConv1D,
13461346
'DepthwiseConv1D': DepthwiseConv1D,
13471347
'SeparableConv2D': SeparableConv2D,
1348+
'QSeparableConv2D': SeparableConv2D,
13481349
'DepthwiseConv2D': DepthwiseConv2D,
13491350
'QDepthwiseConv2D': DepthwiseConv2D,
13501351
'BatchNormalization': BatchNormalization,

‎hls4ml/templates/vitis/nnet_utils/nnet_sepconv1d_stream.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ void pointwise_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
7070
}
7171
}
7272

73-
template <class data_T, class res_T, typename CONFIG_T>
73+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
7474
void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
7575
typename CONFIG_T::depthwise_config::weight_t
7676
depthwise_weights[CONFIG_T::depthwise_config::filt_width * CONFIG_T::depthwise_config::n_chan],
@@ -85,14 +85,14 @@ void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
8585

8686
#pragma HLS DATAFLOW
8787

88-
hls::stream<data_T> depthwise_res;
88+
hls::stream<dw_res_T> depthwise_res;
8989
unsigned res_depth = CONFIG_T::depthwise_config::out_width;
9090
#pragma HLS STREAM variable=depthwise_res depth=res_depth
9191

92-
depthwise_conv_1d_buffer_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
93-
depthwise_biases);
94-
pointwise_conv_1d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
95-
pointwise_biases);
92+
depthwise_conv_1d_buffer_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res,
93+
depthwise_weights, depthwise_biases);
94+
pointwise_conv_1d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
95+
pointwise_biases);
9696
}
9797

9898
} // namespace nnet

‎hls4ml/templates/vitis/nnet_utils/nnet_sepconv2d_stream.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ void depthwise_conv_2d_cl(
103103
depthwise_conv_2d_buffer_cl<data_T, res_T, CONFIG_T>(data, res, weights, biases);
104104
}
105105

106-
template <class data_T, class res_T, typename CONFIG_T>
106+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
107107
void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
108108
typename CONFIG_T::depthwise_config::weight_t
109109
depthwise_weights[CONFIG_T::depthwise_config::filt_height *
@@ -119,14 +119,14 @@ void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
119119

120120
#pragma HLS DATAFLOW
121121

122-
hls::stream<data_T> depthwise_res;
122+
hls::stream<dw_res_T> depthwise_res;
123123
unsigned res_depth = CONFIG_T::depthwise_config::out_height * CONFIG_T::depthwise_config::out_width;
124124
#pragma HLS STREAM variable=depthwise_res depth=res_depth
125125

126-
depthwise_conv_2d_buffer_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
127-
depthwise_biases);
128-
pointwise_conv_2d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
129-
pointwise_biases);
126+
depthwise_conv_2d_buffer_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res,
127+
depthwise_weights, depthwise_biases);
128+
pointwise_conv_2d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
129+
pointwise_biases);
130130
}
131131

132132
} // namespace nnet

‎hls4ml/templates/vivado/nnet_utils/nnet_sepconv1d_stream.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ void pointwise_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
9595
}
9696
}
9797

98-
template <class data_T, class res_T, typename CONFIG_T>
98+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
9999
void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
100100
typename CONFIG_T::depthwise_config::weight_t
101101
depthwise_weights[CONFIG_T::depthwise_config::filt_width * CONFIG_T::depthwise_config::n_chan],
@@ -105,14 +105,14 @@ void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
105105
typename CONFIG_T::pointwise_config::bias_t pointwise_biases[CONFIG_T::pointwise_config::n_filt]) {
106106
#pragma HLS DATAFLOW
107107

108-
hls::stream<data_T> depthwise_res;
108+
hls::stream<dw_res_T> depthwise_res;
109109
unsigned res_depth = CONFIG_T::depthwise_config::out_width;
110110
#pragma HLS STREAM variable=depthwise_res depth=res_depth
111111

112-
depthwise_conv_1d_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
113-
depthwise_biases);
114-
pointwise_conv_1d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
115-
pointwise_biases);
112+
depthwise_conv_1d_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
113+
depthwise_biases);
114+
pointwise_conv_1d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
115+
pointwise_biases);
116116
}
117117

118118
} // namespace nnet

‎hls4ml/templates/vivado/nnet_utils/nnet_sepconv2d_stream.h‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "nnet_common.h"
66
#include "nnet_conv2d_stream.h"
77
#include "nnet_sepconv_stream.h"
8+
#include "nnet_types.h"
89

910
namespace nnet {
1011

@@ -117,7 +118,7 @@ void pointwise_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
117118
}
118119
}
119120

120-
template <class data_T, class res_T, typename CONFIG_T>
121+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
121122
void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
122123
typename CONFIG_T::depthwise_config::weight_t
123124
depthwise_weights[CONFIG_T::depthwise_config::filt_height *
@@ -128,14 +129,14 @@ void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
128129
typename CONFIG_T::pointwise_config::bias_t pointwise_biases[CONFIG_T::pointwise_config::n_filt]) {
129130
#pragma HLS DATAFLOW
130131

131-
hls::stream<data_T> depthwise_res;
132+
hls::stream<dw_res_T> depthwise_res;
132133
unsigned res_depth = CONFIG_T::depthwise_config::out_height * CONFIG_T::depthwise_config::out_width;
133134
#pragma HLS STREAM variable=depthwise_res depth=res_depth
134135

135-
depthwise_conv_2d_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
136-
depthwise_biases);
137-
pointwise_conv_2d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
138-
pointwise_biases);
136+
depthwise_conv_2d_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
137+
depthwise_biases);
138+
pointwise_conv_2d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
139+
pointwise_biases);
139140
}
140141

141142
} // namespace nnet

0 commit comments

Comments
 (0)