Skip to content

Commit 1f9668d

Browse files
vloncarvloncar
committed
Support for QSeparableConv1D/2D
1 parent 4b4b5a0 commit 1f9668d

File tree

7 files changed

+77
-27
lines changed

7 files changed

+77
-27
lines changed

‎hls4ml/backends/vivado/passes/convolution_templates.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,10 +244,12 @@ def __init__(self):
244244
}};\n"""
245245

246246
sepconv1d_function_template = (
247-
'nnet::separable_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {d}, {p}, {z}, {b});'
247+
'nnet::separable_conv_1d_{data_format}<{input_t}, {dw_output_t}, {output_t}, {config}>('
248+
'{input}, {output}, {d}, {p}, {z}, {b});'
248249
)
249250
sepconv2d_function_template = (
250-
'nnet::separable_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {d}, {p}, {z}, {b});'
251+
'nnet::separable_conv_2d_{data_format}<{input_t}, {dw_output_t}, {output_t}, {config}>('
252+
'{input}, {output}, {d}, {p}, {z}, {b});'
251253
)
252254

253255
sepconv1d_include_list = ['nnet_utils/nnet_conv1d.h', 'nnet_utils/nnet_sepconv1d_stream.h']
@@ -360,6 +362,7 @@ def __init__(self):
360362

361363
def format(self, node):
362364
params = self._default_function_params(node)
365+
params['dw_output_t'] = node.get_attr('dw_output_t').name
363366
params['data_format'] = 'cf' if node.get_attr('data_format') == 'channels_first' else 'cl'
364367
params['d'] = node.get_weights('depthwise').name
365368
params['p'] = node.get_weights('pointwise').name
@@ -487,6 +490,7 @@ def __init__(self):
487490

488491
def format(self, node):
489492
params = self._default_function_params(node)
493+
params['dw_output_t'] = node.get_attr('dw_output_t').name
490494
params['data_format'] = 'cf' if node.get_attr('data_format') == 'channels_first' else 'cl'
491495
params['d'] = node.get_weights('depthwise').name
492496
params['p'] = node.get_weights('pointwise').name

‎hls4ml/backends/vivado/vivado_backend.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
Softmax,
2929
)
3030
from hls4ml.model.optimizer import get_backend_passes, layer_optimizer
31-
from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType
31+
from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType, PackedType
3232
from hls4ml.report import parse_vivado_report
3333
from hls4ml.utils.fixed_point_utils import ceil_log2
3434

@@ -75,6 +75,12 @@ def _register_layer_attributes(self):
7575
attrs.append(ChoiceAttribute('conv_implementation', choices=['LineBuffer', 'Encoded'], default='LineBuffer'))
7676
self.attribute_map[layer] = attrs
7777

78+
sep_conv_layers = [SeparableConv1D, SeparableConv2D]
79+
for layer in sep_conv_layers:
80+
attrs = self.attribute_map.get(layer, [])
81+
attrs.append(TypeAttribute('dw_output', default=FixedPrecisionType(18, 8)))
82+
self.attribute_map[layer] = attrs
83+
7884
def _register_flows(self):
7985
initializers = self._get_layer_initializers()
8086
init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
@@ -288,6 +294,15 @@ def init_sepconv1d(self, layer):
288294
) # TODO Once we have SeparableConv implementation for io_parallel this should be set properly
289295
layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
290296

297+
# Set the output type of the depthwise phase
298+
dw_out_precision, _ = layer.model.config.get_precision(layer, 'dw_output')
299+
dw_out_name = layer.name + '_dw_out_t'
300+
if layer.model.config.get_config_value('IOType') == 'io_stream':
301+
dw_output_t = PackedType(dw_out_name, dw_out_precision, layer.get_attr('n_chan'), n_pack=1)
302+
else:
303+
dw_output_t = NamedType(dw_out_name, dw_out_precision)
304+
layer.set_attr('dw_output_t', dw_output_t)
305+
291306
@layer_optimizer(Conv2D)
292307
def init_conv2d(self, layer):
293308
if len(layer.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
@@ -334,6 +349,15 @@ def init_sepconv2d(self, layer):
334349
) # TODO Once we have SeparableConv implementation for io_parallel this should be set properly
335350
layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
336351

352+
# Set the output type of the depthwise phase
353+
dw_out_precision, _ = layer.model.config.get_precision(layer, 'dw_output')
354+
dw_out_name = layer.name + '_dw_out_t'
355+
if layer.model.config.get_config_value('IOType') == 'io_stream':
356+
dw_output_t = PackedType(dw_out_name, dw_out_precision, layer.get_attr('n_chan'), n_pack=1)
357+
else:
358+
dw_output_t = NamedType(dw_out_name, dw_out_precision)
359+
layer.set_attr('dw_output_t', dw_output_t)
360+
337361
@layer_optimizer(DepthwiseConv2D)
338362
def init_depconv2d(self, layer):
339363
if layer.model.config.is_resource_strategy(layer):

‎hls4ml/converters/keras/qkeras.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,27 @@ def parse_qdepthwiseqconv_layer(keras_layer, input_names, input_shapes, data_rea
5454
layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
5555

5656
layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
57+
58+
if keras_layer['config']['bias_quantizer'] is not None:
59+
layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
60+
else:
61+
layer['bias_quantizer'] = None
62+
63+
return layer, output_shape
64+
65+
66+
@keras_handler('QSeparableConv1D', 'QSeparableConv2D')
67+
def parse_qsepconv_layer(keras_layer, input_names, input_shapes, data_reader):
68+
assert 'QSeparableConv' in keras_layer['class_name']
69+
70+
if '1D' in keras_layer['class_name']:
71+
layer, output_shape = parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader)
72+
elif '2D' in keras_layer['class_name']:
73+
layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
74+
75+
layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
76+
layer['pointwise_quantizer'] = get_quantizer_from_config(keras_layer, 'pointwise')
77+
5778
if keras_layer['config']['bias_quantizer'] is not None:
5879
layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
5980
else:

‎hls4ml/templates/vitis/nnet_utils/nnet_sepconv1d_stream.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ void pointwise_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
7070
}
7171
}
7272

73-
template <class data_T, class res_T, typename CONFIG_T>
73+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
7474
void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
7575
typename CONFIG_T::depthwise_config::weight_t
7676
depthwise_weights[CONFIG_T::depthwise_config::filt_width * CONFIG_T::depthwise_config::n_chan],
@@ -85,14 +85,14 @@ void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
8585

8686
#pragma HLS DATAFLOW
8787

88-
hls::stream<data_T> depthwise_res;
88+
hls::stream<dw_res_T> depthwise_res;
8989
unsigned res_depth = CONFIG_T::depthwise_config::out_width;
9090
#pragma HLS STREAM variable=depthwise_res depth=res_depth
9191

92-
depthwise_conv_1d_buffer_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
93-
depthwise_biases);
94-
pointwise_conv_1d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
95-
pointwise_biases);
92+
depthwise_conv_1d_buffer_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res,
93+
depthwise_weights, depthwise_biases);
94+
pointwise_conv_1d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
95+
pointwise_biases);
9696
}
9797

9898
} // namespace nnet

‎hls4ml/templates/vitis/nnet_utils/nnet_sepconv2d_stream.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ void depthwise_conv_2d_cl(
103103
depthwise_conv_2d_buffer_cl<data_T, res_T, CONFIG_T>(data, res, weights, biases);
104104
}
105105

106-
template <class data_T, class res_T, typename CONFIG_T>
106+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
107107
void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
108108
typename CONFIG_T::depthwise_config::weight_t
109109
depthwise_weights[CONFIG_T::depthwise_config::filt_height *
@@ -119,14 +119,14 @@ void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
119119

120120
#pragma HLS DATAFLOW
121121

122-
hls::stream<data_T> depthwise_res;
122+
hls::stream<dw_res_T> depthwise_res;
123123
unsigned res_depth = CONFIG_T::depthwise_config::out_height * CONFIG_T::depthwise_config::out_width;
124124
#pragma HLS STREAM variable=depthwise_res depth=res_depth
125125

126-
depthwise_conv_2d_buffer_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
127-
depthwise_biases);
128-
pointwise_conv_2d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
129-
pointwise_biases);
126+
depthwise_conv_2d_buffer_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res,
127+
depthwise_weights, depthwise_biases);
128+
pointwise_conv_2d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
129+
pointwise_biases);
130130
}
131131

132132
} // namespace nnet

‎hls4ml/templates/vivado/nnet_utils/nnet_sepconv1d_stream.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ void pointwise_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
9595
}
9696
}
9797

98-
template <class data_T, class res_T, typename CONFIG_T>
98+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
9999
void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
100100
typename CONFIG_T::depthwise_config::weight_t
101101
depthwise_weights[CONFIG_T::depthwise_config::filt_width * CONFIG_T::depthwise_config::n_chan],
@@ -105,14 +105,14 @@ void separable_conv_1d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
105105
typename CONFIG_T::pointwise_config::bias_t pointwise_biases[CONFIG_T::pointwise_config::n_filt]) {
106106
#pragma HLS DATAFLOW
107107

108-
hls::stream<data_T> depthwise_res;
108+
hls::stream<dw_res_T> depthwise_res;
109109
unsigned res_depth = CONFIG_T::depthwise_config::out_width;
110110
#pragma HLS STREAM variable=depthwise_res depth=res_depth
111111

112-
depthwise_conv_1d_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
113-
depthwise_biases);
114-
pointwise_conv_1d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
115-
pointwise_biases);
112+
depthwise_conv_1d_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
113+
depthwise_biases);
114+
pointwise_conv_1d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
115+
pointwise_biases);
116116
}
117117

118118
} // namespace nnet

‎hls4ml/templates/vivado/nnet_utils/nnet_sepconv2d_stream.h‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "nnet_common.h"
66
#include "nnet_conv2d_stream.h"
77
#include "nnet_sepconv_stream.h"
8+
#include "nnet_types.h"
89

910
namespace nnet {
1011

@@ -117,7 +118,7 @@ void pointwise_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
117118
}
118119
}
119120

120-
template <class data_T, class res_T, typename CONFIG_T>
121+
template <class data_T, class dw_res_T, class res_T, typename CONFIG_T>
121122
void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
122123
typename CONFIG_T::depthwise_config::weight_t
123124
depthwise_weights[CONFIG_T::depthwise_config::filt_height *
@@ -128,14 +129,14 @@ void separable_conv_2d_cl(hls::stream<data_T> &data, hls::stream<res_T> &res,
128129
typename CONFIG_T::pointwise_config::bias_t pointwise_biases[CONFIG_T::pointwise_config::n_filt]) {
129130
#pragma HLS DATAFLOW
130131

131-
hls::stream<data_T> depthwise_res;
132+
hls::stream<dw_res_T> depthwise_res;
132133
unsigned res_depth = CONFIG_T::depthwise_config::out_height * CONFIG_T::depthwise_config::out_width;
133134
#pragma HLS STREAM variable=depthwise_res depth=res_depth
134135

135-
depthwise_conv_2d_cl<data_T, data_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
136-
depthwise_biases);
137-
pointwise_conv_2d_cl<data_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
138-
pointwise_biases);
136+
depthwise_conv_2d_cl<data_T, dw_res_T, typename CONFIG_T::depthwise_config>(data, depthwise_res, depthwise_weights,
137+
depthwise_biases);
138+
pointwise_conv_2d_cl<dw_res_T, res_T, typename CONFIG_T::pointwise_config>(depthwise_res, res, pointwise_weights,
139+
pointwise_biases);
139140
}
140141

141142
} // namespace nnet

0 commit comments

Comments
 (0)