Skip to content

Commit cf91c3b

Browse files
bo3zbo3z
authored
Merge branch 'main' into attrs_desc
2 parents 88c1fe7 + 2fc8941 commit cf91c3b

File tree

21 files changed

+559
-64
lines changed

21 files changed

+559
-64
lines changed

‎example-models‎

‎hls4ml/backends/fpga/passes/codegen.py‎ renamed to ‎hls4ml/backends/fpga/passes/im2col_codegen.py‎

File renamed without changes.

‎hls4ml/backends/vivado/passes/convolution_templates.py‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
typedef {config_t} mult_config;
6161
template<unsigned K, unsigned S, unsigned W>
6262
using scale_index = nnet::{scale_index_type}<K, S, W>;
63+
template<class data_T, class res_T, class CONFIG_T>
64+
using conv_kernel = nnet::{conv_fn}<data_T, res_T, CONFIG_T>;
6365
}};
6466
const ap_uint<config{index}::filt_width> config{index}::pixels[] = {{{instructions}}};\n"""
6567

@@ -93,11 +95,30 @@ def format(self, node):
9395
else:
9496
params['fill_fn'] = 'FillConv1DBuffer'
9597

98+
is_pointwise_parallel_latency = (
99+
node.get_attr('filt_width') == 1
100+
and node.get_attr('strategy').lower() == 'latency'
101+
and node.model.config.get_config_value('IOType') == 'io_parallel'
102+
)
103+
if is_pointwise_parallel_latency:
104+
params['conv_fn'] = f'pointwise_conv_{node.index}'
105+
else:
106+
if node.get_attr('strategy').lower() == 'latency':
107+
params['conv_fn'] = 'Conv1DLatency'
108+
else:
109+
params['conv_fn'] = 'Conv1DResource'
110+
96111
conv_config = self.template.format(**params)
97112

98113
mult_params = self._default_config_params(node)
99-
mult_params['n_in'] = node.get_attr('n_chan') * node.get_attr('filt_width')
100-
mult_params['n_out'] = node.get_attr('n_filt')
114+
if is_pointwise_parallel_latency:
115+
mult_params['n_in'] = int(
116+
node.get_attr('in_width') * node.get_attr('n_chan') * node.get_attr('filt_width') / mult_params['reuse']
117+
)
118+
mult_params['n_out'] = int(node.get_attr('in_width') * node.get_attr('n_filt') / mult_params['reuse'])
119+
else:
120+
mult_params['n_in'] = node.get_attr('n_chan') * node.get_attr('filt_width')
121+
mult_params['n_out'] = node.get_attr('n_filt')
101122
mult_params['nzeros'] = node.get_weights('weight').nzeros
102123
mult_params['product_type'] = get_backend('vivado').product_type(
103124
node.get_input_variable().type.precision, node.get_weights('weight').type.precision

‎hls4ml/backends/vivado/passes/pointwise_codegen.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
from hls4ml.model.layers import Conv1D
2+
from hls4ml.model.optimizer import OptimizerPass
3+
from hls4ml.model.types import Source
4+
5+
6+
def generate_pointwise_conv1d_fn(layer_idx, reuse_factor=1):
7+
"""Generate a C++ function for a pointwise convolution layer.
8+
9+
Args:
10+
layer_idx (int): Index of layer ('index' attribute).
11+
reuse_factor (int): Number of partitions to divide the input into.
12+
13+
Returns:
14+
str: Generated C++ function
15+
"""
16+
17+
generated_code = (
18+
'template<class data_T, class res_T, typename CONFIG_T>\n'
19+
'class pointwise_conv_{index} : public Conv1DKernel<data_T, res_T, CONFIG_T> {{\n'
20+
' public:\n'
21+
' static void conv(\n'
22+
' data_T data[CONFIG_T::in_width * CONFIG_T::n_chan],\n'
23+
' res_T res[CONFIG_T::out_width * CONFIG_T::n_filt],\n'
24+
' typename CONFIG_T::weight_t weights[CONFIG_T::n_chan * CONFIG_T::n_filt],\n'
25+
' typename CONFIG_T::bias_t biases[CONFIG_T::n_filt]) {{\n'
26+
' data_T data_tmp[CONFIG_T::reuse_factor][CONFIG_T::in_width * CONFIG_T::n_chan / CONFIG_T::reuse_factor];\n' # noqa: E501
27+
' #pragma HLS ARRAY_PARTITION variable=data_tmp complete dim=0\n'
28+
' res_T res_tmp[CONFIG_T::reuse_factor][CONFIG_T::out_width * CONFIG_T::n_filt / CONFIG_T::reuse_factor];\n' # noqa: E501
29+
' #pragma HLS ARRAY_PARTITION variable=res_tmp complete dim=0\n\n'
30+
' RFInputLoop:\n'
31+
' for (int jj = 0; jj < CONFIG_T::reuse_factor; jj++) {{\n'
32+
' #pragma HLS UNROLL\n'
33+
' InnerInputLoop:\n'
34+
' for (int ii = 0; ii < CONFIG_T::in_width * CONFIG_T::n_chan / CONFIG_T::reuse_factor; ii++) {{\n'
35+
' #pragma HLS UNROLL\n'
36+
' data_tmp[jj][ii] = data[jj * CONFIG_T::in_width * CONFIG_T::n_chan / CONFIG_T::reuse_factor + ii];\n' # noqa: E501
37+
' }}\n'
38+
' }}\n\n'
39+
).format(index=layer_idx)
40+
indent = ' '
41+
for i in range(reuse_factor):
42+
generated_code += indent
43+
generated_code += (
44+
f'pointwise_conv_1d_latency_cl<data_T, res_T, CONFIG_T>(data_tmp[{i}], res_tmp[{i}], weights, biases);\n'
45+
)
46+
47+
generated_code += (
48+
'\n'
49+
' RFOutputLoop:\n'
50+
' for (int jj = 0; jj < CONFIG_T::reuse_factor; jj++) {\n'
51+
' #pragma HLS UNROLL\n'
52+
' InnerOutputLoop:\n'
53+
' for (int ii = 0; ii < CONFIG_T::out_width * CONFIG_T::n_filt / CONFIG_T::reuse_factor; ii++) {\n'
54+
' #pragma HLS UNROLL\n'
55+
' res[jj * CONFIG_T::out_width * CONFIG_T::n_filt / CONFIG_T::reuse_factor + ii] = res_tmp[jj][ii];\n' # noqa: E501
56+
' }\n'
57+
' }\n'
58+
' }\n'
59+
'};\n'
60+
)
61+
62+
return generated_code
63+
64+
65+
class GeneratePointwiseConv1D(OptimizerPass):
66+
'''Generates code for pointwise 1D convolution'''
67+
68+
def match(self, node):
69+
return (
70+
isinstance(node, Conv1D)
71+
and node.model.config.get_config_value('IOType') == 'io_parallel'
72+
and node.get_attr('filt_width') == 1
73+
)
74+
75+
def transform(self, model, node):
76+
self._generate_pointwise_conv1d(node)
77+
78+
def _generate_pointwise_conv1d(self, node):
79+
code_str = generate_pointwise_conv1d_fn(
80+
node.get_attr('index'),
81+
node.get_attr('reuse_factor'),
82+
)
83+
84+
node.set_attr('pointwise_conv1d_codegen', Source(code_str))

‎hls4ml/backends/vivado/vivado_backend.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ def _register_flows(self):
123123
'vivado:generate_conv_streaming_instructions',
124124
'vivado:apply_resource_strategy',
125125
'vivado:generate_conv_im2col',
126+
'vivado:generate_pointwise_conv1_d',
126127
'vivado:generate_unrolled_dense_resource',
127128
'vivado:set_pipeline_style',
128129
]

‎hls4ml/converters/onnx/reshape.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from hls4ml.converters.onnx_to_hls import onnx_handler
1+
from hls4ml.converters.onnx_to_hls import get_onnx_attribute, onnx_handler
22

33

44
@onnx_handler('Transpose')
@@ -36,3 +36,25 @@ def parse_flatten_layer(node, input_names, input_shapes, graph):
3636
layer['target_shape'] = [-1] # does not contain batch dimension
3737

3838
return layer
39+
40+
41+
@onnx_handler('Resize')
42+
def parse_resize_layer(node, input_names, input_shapes, graph):
43+
layer = {}
44+
layer['name'] = node.name
45+
layer['class_name'] = 'Resize'
46+
layer['inputs'] = input_names
47+
layer['outputs'] = list(node.output)
48+
layer['in_height'] = input_shapes[0][2]
49+
layer['in_width'] = input_shapes[0][1]
50+
layer['out_width'] = input_shapes[0][1]
51+
layer['out_height'] = input_shapes[0][2]
52+
layer['n_chan'] = input_shapes[0][3]
53+
layer['algorithm'] = get_onnx_attribute(node, 'mode')
54+
# The following is used in initialize() method.
55+
# Probably a better solution would be to have a channels last parameter at QONNX level
56+
layer['data_format'] = (
57+
'channels_last' if any(node.domain == 'qonnx.custom_op.channels_last' for node in graph.node) else 'channels_first'
58+
)
59+
60+
return layer

‎hls4ml/model/layers.py‎

Lines changed: 60 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1148,20 +1148,67 @@ class Resize(Layer):
11481148
def initialize(self):
11491149
inp = self.get_input_variable()
11501150

1151-
if self.get_attr('data_format') == 'channels_last':
1152-
if len(inp.shape) == 2: # 1D -> width + chan
1153-
shape = [self.get_attr('out_width'), self.get_attr('n_chan')]
1154-
dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
1155-
elif len(inp.shape) == 3: # 2D -> height + width + chan
1156-
shape = [self.get_attr('out_height'), self.get_attr('out_width'), self.get_attr('n_chan')]
1157-
dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
1151+
if len(self.inputs) > 1:
1152+
# In order to be correctly ingested by hls4ml the QONNX resize node should have 3 inputs set with RoI left empty
1153+
if len(self.inputs) == 2:
1154+
raise Exception(
1155+
'The number of inputs to Resize node is equal to 2. '
1156+
'In this case, either one is trying to use a version 10 node '
1157+
'or one is using the RoI parameter only to perform the resize operation, '
1158+
'both not supported in hls4ml'
1159+
)
1160+
if len(self.inputs) == 4:
1161+
raise Exception('Sizes parameter is not supported by hls4ml. Use scales instead')
1162+
# get the scales of Resize node from QONNX frontend
1163+
# see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html
1164+
scales_idx = 2 if len(self.inputs) == 3 or len(self.inputs) == 4 else 1
1165+
scales = self.get_input_node(self.inputs[scales_idx]).get_attr('value')
1166+
if len(scales) == 4: # Resize 2D
1167+
self.set_attr('out_width', int(self.get_attr('in_width') * scales[1]))
1168+
self.set_attr('out_height', int(self.get_attr('in_height') * scales[2]))
1169+
self.set_attr('n_chan', int(self.get_attr('n_chan') * scales[3]))
1170+
elif len(scales) == 3: # Resize 1D
1171+
self.set_attr('out_width', int(self.get_attr('in_width') * scales[1]))
1172+
self.set_attr('n_chan', int(self.get_attr('n_chan') * scales[2]))
1173+
else:
1174+
raise Exception('Resize 1D and Resize 2D are the ones supported in hls4ml')
1175+
if self.get_attr('data_format') == 'channels_last':
1176+
if len(inp.shape) == 2: # 1D -> width + chan
1177+
shape = [int(self.get_attr('out_width')), int(self.get_attr('n_chan'))]
1178+
dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
1179+
elif len(inp.shape) == 3: # 2D -> height + width + chan
1180+
shape = [
1181+
int(self.get_attr('out_height')),
1182+
int(self.get_attr('out_width')),
1183+
int(self.get_attr('n_chan')),
1184+
]
1185+
dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
1186+
else:
1187+
if len(inp.shape) == 2: # 1D -> width + chan
1188+
shape = [int(self.get_attr('n_chan')), int(self.get_attr('out_width'))]
1189+
dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}']
1190+
elif len(inp.shape) == 3: # 2D -> height + width + chan
1191+
shape = [
1192+
int(self.get_attr('n_chan')),
1193+
int(self.get_attr('out_height')),
1194+
int(self.get_attr('out_width')),
1195+
]
1196+
dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}']
11581197
else:
1159-
if len(inp.shape) == 2: # 1D -> width + chan
1160-
shape = [self.get_attr('n_chan'), self.get_attr('out_width')]
1161-
dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}']
1162-
elif len(inp.shape) == 3: # 2D -> height + width + chan
1163-
shape = [self.get_attr('n_chan'), self.get_attr('out_height'), self.get_attr('out_width')]
1164-
dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}']
1198+
if self.get_attr('data_format') == 'channels_last':
1199+
if len(inp.shape) == 2: # 1D -> width + chan
1200+
shape = [self.get_attr('out_width'), self.get_attr('n_chan')]
1201+
dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
1202+
elif len(inp.shape) == 3: # 2D -> height + width + chan
1203+
shape = [self.get_attr('out_height'), self.get_attr('out_width'), self.get_attr('n_chan')]
1204+
dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
1205+
else:
1206+
if len(inp.shape) == 2: # 1D -> width + chan
1207+
shape = [self.get_attr('n_chan'), self.get_attr('out_width')]
1208+
dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}']
1209+
elif len(inp.shape) == 3: # 2D -> height + width + chan
1210+
shape = [self.get_attr('n_chan'), self.get_attr('out_height'), self.get_attr('out_width')]
1211+
dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}']
11651212

11661213
self.add_output_variable(shape, dims, precision=inp.type.precision)
11671214

‎hls4ml/model/optimizer/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
'parse_qonnx',
3535
[
3636
'reshape_constant',
37+
'resize_remove_constants',
3738
'quant_constant_parameters',
3839
'quant_to_activation',
3940
'fuse_quant_with_constant',

‎hls4ml/model/optimizer/passes/resize_remove_constants.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
from warnings import warn
2+
3+
from hls4ml.model.layers import Constant, Resize
4+
from hls4ml.model.optimizer import OptimizerPass
5+
6+
7+
class ResizeRemoveConstants(OptimizerPass):
8+
"""
9+
This optimizer is intended to clean the Resize node from RoI and Scales parameters that if left cause issues in hls4ml.
10+
"""
11+
12+
def match(self, node):
13+
is_match = isinstance(node, Resize) and len(node.inputs) > 1
14+
return is_match
15+
16+
def transform(self, model, node):
17+
"""
18+
Remove RoI and Scale Constant from new shape input.
19+
"""
20+
# see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html
21+
roi_index = 1
22+
scales_idx = 2
23+
scales_node = node.get_input_node(node.inputs[scales_idx])
24+
node.inputs[scales_idx] = ''
25+
if not isinstance(scales_node, Constant):
26+
raise RuntimeError("Non-constant shape inputs are not supported")
27+
model.remove_node(scales_node, rewire=False)
28+
# RoI position is always 1 when present
29+
roi_node = node.get_input_node(node.inputs[roi_index])
30+
if roi_node.get_attr('value'):
31+
warn('RoI value vector is not empty. Consider that RoI is not supported in hls4ml', stacklevel=2)
32+
node.inputs[roi_index] = ''
33+
if not isinstance(roi_node, Constant):
34+
raise RuntimeError("Non-constant RoI inputs are not supported")
35+
model.remove_node(roi_node, rewire=False)
36+
# Clean all the '' inputs
37+
node.inputs = list(filter(None, node.inputs))
38+
return True

‎hls4ml/templates/vitis/nnet_utils/nnet_conv1d.h‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "nnet_common.h"
55
#include "nnet_conv1d_latency.h"
66
#include "nnet_conv1d_resource.h"
7+
#include "nnet_function_stubs.h"
78
#include <cstdlib>
89

910
namespace nnet {
@@ -38,11 +39,7 @@ void conv_1d_cl(data_T data[CONFIG_T::in_width * CONFIG_T::n_chan], res_T res[CO
3839
// Inlining helps reduce latency, but may also cause timing issues in some cases, use carefully.
3940
//#pragma HLS INLINE recursive
4041

41-
if (CONFIG_T::strategy == nnet::latency) {
42-
conv_1d_latency_cl<data_T, res_T, CONFIG_T>(data, res, weights, biases);
43-
} else {
44-
conv_1d_resource_cl<data_T, res_T, CONFIG_T>(data, res, weights, biases);
45-
}
42+
CONFIG_T::template conv_kernel<data_T, res_T, CONFIG_T>::conv(data, res, weights, biases);
4643
}
4744

4845
template <class data_T, class res_T, typename CONFIG_T>
@@ -55,13 +52,28 @@ void pointwise_conv_1d_cl(data_T data[CONFIG_T::in_width * CONFIG_T::n_chan],
5552
// Inlining helps reduce latency, but may also cause timing issues in some cases, use carefully.
5653
//#pragma HLS INLINE recursive
5754

58-
// Nothing special to be done for io_parallel implementation
59-
if (CONFIG_T::strategy == nnet::latency) {
55+
CONFIG_T::template conv_kernel<data_T, res_T, CONFIG_T>::conv(data, res, weights, biases);
56+
}
57+
58+
template <class data_T, class res_T, typename CONFIG_T> class Conv1DLatency : public Conv1DKernel<data_T, res_T, CONFIG_T> {
59+
public:
60+
static void conv(data_T data[CONFIG_T::in_width * CONFIG_T::n_chan], res_T res[CONFIG_T::out_width * CONFIG_T::n_filt],
61+
typename CONFIG_T::weight_t weights[CONFIG_T::filt_width * CONFIG_T::n_chan * CONFIG_T::n_filt],
62+
typename CONFIG_T::bias_t biases[CONFIG_T::n_filt]) {
63+
//#pragma HLS INLINE region
6064
conv_1d_latency_cl<data_T, res_T, CONFIG_T>(data, res, weights, biases);
61-
} else {
65+
}
66+
};
67+
68+
template <class data_T, class res_T, typename CONFIG_T> class Conv1DResource : public Conv1DKernel<data_T, res_T, CONFIG_T> {
69+
public:
70+
static void conv(data_T data[CONFIG_T::in_width * CONFIG_T::n_chan], res_T res[CONFIG_T::out_width * CONFIG_T::n_filt],
71+
typename CONFIG_T::weight_t weights[CONFIG_T::filt_width * CONFIG_T::n_chan * CONFIG_T::n_filt],
72+
typename CONFIG_T::bias_t biases[CONFIG_T::n_filt]) {
73+
//#pragma HLS INLINE region
6274
conv_1d_resource_cl<data_T, res_T, CONFIG_T>(data, res, weights, biases);
6375
}
64-
}
76+
};
6577

6678
} // namespace nnet
6779

0 commit comments

Comments
 (0)