Merge branch 'main' into fix_repack_precision

Author: calad0i

.pre-commit-config.yaml

@@ -2,15 +2,15 @@ exclude: (^hls4ml\/templates\/(vivado|quartus)\/(ap_types|ac_types)\/|^test/pyte
 
 repos:
 - repo: https://github.com/psf/black
-  rev: 23.9.1
+  rev: 23.10.0
   hooks:
   - id: black
     language_version: python3
     args: ['--line-length=125',
            '--skip-string-normalization']
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -30,7 +30,7 @@ repos:
     args: ["--profile", "black", --line-length=125]
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.14.0
+  rev: v3.15.0
   hooks:
   - id: pyupgrade
     args: ["--py36-plus"]

hls4ml/backends/vivado/passes/convolution_templates.py

@@ -41,6 +41,8 @@
     static const unsigned out_width = {out_width};
     static const unsigned reuse_factor = {reuse};
     static const unsigned n_zeros = {nzeros};
+    static const unsigned multiplier_limit =
+        DIV_ROUNDUP(kernel_size * n_chan * n_filt, reuse_factor) - n_zeros / reuse_factor;
     static const bool store_weights_in_bram = false;
     static const unsigned strategy = nnet::{strategy};
     static const nnet::conv_implementation implementation = nnet::conv_implementation::{implementation};

hls4ml/model/profiling.py

@@ -10,7 +10,7 @@
 import seaborn as sb
 
 from hls4ml.model.graph import ModelGraph
-from hls4ml.model.layers import GRU, LSTM
+from hls4ml.model.layers import GRU, LSTM, SeparableConv1D, SeparableConv2D
 
 try:
     import qkeras
@@ -184,6 +184,8 @@ def types_hlsmodel(model):
     for layer in model.get_layers():
         if isinstance(layer, GRU) or isinstance(layer, LSTM):
             suffix = ['w', 'rw', 'b', 'rb']
+        elif isinstance(layer, SeparableConv1D) or isinstance(layer, SeparableConv2D):
+            suffix = ['dw', 'pw', 'db', 'pb']
         else:
             suffix = ['w', 'b']
         for iw, weight in enumerate(layer.get_weights()):
@@ -225,6 +227,8 @@ def weights_hlsmodel(model, fmt='longform', plot='boxplot'):
     for layer in model.get_layers():
         if isinstance(layer, GRU) or isinstance(layer, LSTM):
             suffix = ['w', 'rw', 'b', 'rb']
+        elif isinstance(layer, SeparableConv1D) or isinstance(layer, SeparableConv2D):
+            suffix = ['dw', 'pw', 'db', 'pb']
         else:
             suffix = ['w', 'b']
         name = layer.name
@@ -343,21 +347,23 @@ def activations_keras(model, X, fmt='longform', plot='boxplot'):
         # return summary statistics for matplotlib.axes.Axes.bxp
         # or histogram bin edges and heights
         data = []
-
-    for layer in model.layers:
-        print(f"   {layer.name}")
-        if not isinstance(layer, keras.layers.InputLayer):
-            y = _get_output(layer, X, model.input).flatten()
-            y = abs(y[y != 0])
-            if len(y) == 0:
-                print(f'Activations for {layer.name} are only zeros, ignoring.')
-                continue
-            if fmt == 'longform':
-                data['x'].extend(y.tolist())
-                data['weight'].extend([layer.name for i in range(len(y))])
-            elif fmt == 'summary':
-                data.append(array_to_summary(y, fmt=plot))
-                data[-1]['weight'] = layer.name
+    outputs = _get_outputs(
+        [layer for layer in model.layers if not isinstance(layer, keras.layers.InputLayer)], X, model.input
+    )
+    outputs = dict(zip([layer.name for layer in model.layers if not isinstance(layer, keras.layers.InputLayer)], outputs))
+    for layer_name, y in outputs.items():
+        print(f"   {layer_name}")
+        y = y.flatten()
+        y = abs(y[y != 0])
+        if len(y) == 0:
+            print(f'Activations for {layer_name} are only zeros, ignoring.')
+            continue
+        if fmt == 'longform':
+            data['x'].extend(y.tolist())
+            data['weight'].extend([layer_name for i in range(len(y))])
+        elif fmt == 'summary':
+            data.append(array_to_summary(y, fmt=plot))
+            data[-1]['weight'] = layer_name
 
     if fmt == 'longform':
         data = pandas.DataFrame(data)
@@ -544,10 +550,10 @@ def _is_ignored_layer(layer):
     return False
 
 
-def _get_output(layer, X, model_input):
-    """Get output of partial model"""
-    partial_model = keras.models.Model(inputs=model_input, outputs=layer.output)
-    y = partial_model.predict(X)
+def _get_outputs(layers, X, model_input):
+    """Get outputs of intermediate layers"""
+    partial_models = keras.models.Model(inputs=model_input, outputs=[layer.output for layer in layers])
+    y = partial_models.predict(X)
     return y
 
 
@@ -562,37 +568,30 @@ def get_ymodel_keras(keras_model, X):
     Returns:
         dict: A dictionary in the form {"layer_name": ouput array of layer}.
     """
-
     ymodel = {}
-
+    traced_layers = []
+    layer_names = []
     for layer in keras_model.layers:
-        print(f"Processing {layer.name} in Keras model...")
-        if not _is_ignored_layer(layer):
-            # If the layer has activation integrated then separate them
-            # Note that if the layer is a standalone activation layer then skip this
-            if hasattr(layer, 'activation') and not (
-                isinstance(layer, keras.layers.Activation) or isinstance(layer, qkeras.qlayers.QActivation)
-            ):
-                if layer.activation:
-                    if layer.activation.__class__.__name__ == "linear":
-                        ymodel[layer.name] = _get_output(layer, X, keras_model.input)
-
-                    else:
-                        temp_activation = layer.activation
-                        layer.activation = None
-                        # Get output for layer without activation
-                        ymodel[layer.name] = _get_output(layer, X, keras_model.input)
-
-                        # Add the activation back
-                        layer.activation = temp_activation
-                        # Get ouput for activation
-                        ymodel[layer.name + f"_{temp_activation.__class__.__name__}"] = _get_output(
-                            layer, X, keras_model.input
-                        )
-                else:
-                    ymodel[layer.name] = _get_output(layer, X, keras_model.input)
-            else:
-                ymodel[layer.name] = _get_output(layer, X, keras_model.input)
+        if _is_ignored_layer(layer):
+            continue
+        # If the layer has activation integrated then separate them
+        # Note that if the layer is a standalone activation layer then skip this
+        name = layer.name
+        if (
+            hasattr(layer, "activation")
+            and layer.activation.__name__ != "linear"
+            and not isinstance(layer, (keras.layers.Activation, qkeras.qlayers.QActivation))
+        ):
+            tmp_activation = layer.activation
+            layer.activation = None
+            ymodel.update({layer.name: _get_outputs([layer], X, keras_model.input)})
+            layer.activation = tmp_activation
+            name = layer.name + f"_{tmp_activation.__name__}"
+        traced_layers.append(layer)
+        layer_names.append(name)
+    outputs = _get_outputs(traced_layers, X, keras_model.input)
+    for name, output in zip(layer_names, outputs):
+        ymodel[name] = output
     print("Done taking outputs for Keras model.")
     return ymodel
 

hls4ml/templates/quartus/firmware/nnet_utils/nnet_conv2d_stream.h

@@ -69,7 +69,7 @@ template <class data_T, typename CONFIG_T>
 void shift_line_buffer_2d(
     const data_T &in_elem,
     nnet::shift_reg<typename data_T::value_type, CONFIG_T::pad_left + CONFIG_T::in_width + CONFIG_T::pad_right>
-        line_buffer[CONFIG_T::filt_height - 1][CONFIG_T::n_chan],
+        line_buffer[MAX(CONFIG_T::filt_height - 1, 1)][CONFIG_T::n_chan],
     typename data_T::value_type shift_buffer[CONFIG_T::filt_height][CONFIG_T::n_chan]) {
 // For every channel, insert the incoming pixel at end of the shift buffer
 UpdateBuffer:

test/pytest/test_pointwiseconv.py

@@ -11,10 +11,8 @@
 
 padds_options = ['same', 'valid']
 chans_options = ['channels_last']
-io_type_options = ['io_parallel', 'io_stream']
 strides1d_options = [(1,), (2,)]
 strides2d_options = [(1, 1), (2, 2)]
-strategy_options = ['Latency', 'Resource']
 
 
 @pytest.mark.parametrize('chans', chans_options)
@@ -24,6 +22,7 @@
     'backend, io_type, strategy',
     [
         ('Quartus', 'io_parallel', 'resource'),
+        ('Quartus', 'io_stream', 'resource'),
         ('Vivado', 'io_parallel', 'resource'),
         ('Vitis', 'io_parallel', 'resource'),
         ('Vivado', 'io_parallel', 'latency'),
@@ -54,7 +53,7 @@ def test_pointwiseconv1d(chans, padds, strides, backend, io_type, strategy):
     X_input = np.random.rand(100, *input_shape)
     keras_prediction = model.predict(X_input)
 
-    default_precision = 'ac_fixed<32,16,true>' if backend == 'Quartus' else 'ap_fixed<32,16>'
+    default_precision = 'fixed<32,16>'
     config = hls4ml.utils.config_from_keras_model(model, default_precision=default_precision)
     config['Model']['Strategy'] = strategy
 
@@ -70,7 +69,9 @@ def test_pointwiseconv1d(chans, padds, strides, backend, io_type, strategy):
     hls_model.compile()
     hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape)
 
-    assert 'Pointwise' in list(hls_model.graph.values())[1].class_name
+    if not (backend == 'Quartus' and io_type == 'io_stream'):
+        # Quartus io_stream does not currently have a special pointwise implementation
+        assert 'Pointwise' in list(hls_model.graph.values())[1].class_name
     np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=0.001)
 
 
@@ -81,6 +82,7 @@ def test_pointwiseconv1d(chans, padds, strides, backend, io_type, strategy):
     'backend, io_type, strategy',
     [
         ('Quartus', 'io_parallel', 'resource'),
+        ('Quartus', 'io_stream', 'resource'),
         ('Vivado', 'io_parallel', 'resource'),
         ('Vivado', 'io_parallel', 'latency'),
         ('Vivado', 'io_stream', 'latency'),
@@ -107,7 +109,7 @@ def test_pointwiseconv2d(chans, padds, strides, backend, io_type, strategy):
     X_input = np.random.rand(100, *input_shape)
     keras_prediction = model.predict(X_input)
 
-    default_precision = 'ac_fixed<32, 9, true>' if backend == 'Quartus' else 'ap_fixed<32, 9>'
+    default_precision = 'fixed<32, 9>'
 
     config = hls4ml.utils.config_from_keras_model(model, default_precision=default_precision)
     config['Model']['Strategy'] = strategy
@@ -125,7 +127,9 @@ def test_pointwiseconv2d(chans, padds, strides, backend, io_type, strategy):
     hls_model.compile()
     hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape)
 
-    assert 'Pointwise' in list(hls_model.graph.values())[1].class_name
+    if not (backend == 'Quartus' and io_type == 'io_stream'):
+        # Quartus io_stream does not currently have a special pointwise implementation
+        assert 'Pointwise' in list(hls_model.graph.values())[1].class_name
     np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=0.001)
 
 

test/pytest/test_sepconv1d.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+
+import numpy as np
+import pytest
+import tensorflow as tf
+from tensorflow.keras.layers import SeparableConv1D
+
+import hls4ml
+
+test_root_path = Path(__file__).parent
+
+keras_conv1d = [SeparableConv1D]
+padds_options = ['same', 'valid']
+chans_options = ['channels_last']
+io_type_options = ['io_stream']
+strides_options = [(1), (2)]
+kernel_options = [(1), (3)]
+bias_options = [False]
+
+
+@pytest.mark.parametrize('conv1d', keras_conv1d)
+@pytest.mark.parametrize('chans', chans_options)
+@pytest.mark.parametrize('padds', padds_options)
+@pytest.mark.parametrize('strides', strides_options)
+@pytest.mark.parametrize('kernels', kernel_options)
+@pytest.mark.parametrize('bias', bias_options)
+@pytest.mark.parametrize('io_type', io_type_options)
+@pytest.mark.parametrize('backend', ['Vivado', 'Vitis'])
+def test_sepconv1d(conv1d, chans, padds, strides, kernels, bias, io_type, backend):
+    model = tf.keras.models.Sequential()
+    input_shape = (28, 3)
+    model.add(
+        conv1d(
+            filters=32,
+            kernel_size=kernels,
+            strides=strides,
+            padding=padds,
+            input_shape=input_shape,
+            kernel_initializer='normal',
+            use_bias=bias,
+            data_format=chans,
+        )
+    )
+
+    model.compile(optimizer='adam', loss='mse')
+    X_input = np.random.rand(100, *input_shape)
+    keras_prediction = model.predict(X_input)
+    config = hls4ml.utils.config_from_keras_model(model, default_precision='ap_fixed<32,16>')
+    stride_cfg = str(strides).replace(', ', '_').replace('(', '').replace(')', '')
+    kernel_cfg = str(kernels).replace(', ', '_').replace('(', '').replace(')', '')
+    output_dir = str(
+        test_root_path
+        / 'hls4mlprj_{}_{}_strides_{}_kernels_{}_{}_padding_{}_{}'.format(
+            conv1d.__name__.lower(), chans, stride_cfg, kernel_cfg, padds, backend, io_type
+        )
+    )
+    hls_model = hls4ml.converters.convert_from_keras_model(
+        model, hls_config=config, output_dir=output_dir, io_type=io_type, backend=backend
+    )
+    hls_model.compile()
+    hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape)
+
+    np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=0.001)

test/pytest/test_trace.py

@@ -12,14 +12,16 @@
 
 
 @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus'])
-def test_trace(backend):
+@pytest.mark.parametrize('activation', ['relu', None])
+def test_trace(backend, activation):
     '''Test the tracing feature with a simple Keras model.'''
     model = tf.keras.models.Sequential()
     model.add(
         Dense(
             2,
             input_shape=(1,),
             name='Dense',
+            activation=activation,
             use_bias=True,
             kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10),
             bias_initializer='zeros',
@@ -48,6 +50,7 @@ def test_trace(backend):
     hls_model.compile()
     hls4ml_pred, hls4ml_trace = hls_model.trace(X_input)
     keras_trace = hls4ml.model.profiling.get_ymodel_keras(model, X_input)
-
-    np.testing.assert_allclose(hls4ml_trace['Dense'], keras_trace['Dense'], rtol=1e-2, atol=0.01)
+    assert keras_trace.keys() == hls4ml_trace.keys()
+    for key in hls4ml_trace.keys():
+        np.testing.assert_allclose(hls4ml_trace[key], keras_trace[key], rtol=1e-2, atol=0.01)
     np.testing.assert_allclose(hls4ml_pred, keras_prediction, rtol=1e-2, atol=0.01)