Merge branch 'main' into hls4ml-optimization-api-part-1

Author: bo3z

.gitlab-ci.yml

@@ -7,7 +7,7 @@ generator:
   stage: generate
   image: python:3.8-alpine
   tags:
-    - docker
+    - k8s-default
   before_script:
     - pip install pyyaml
   script:

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: (^hls4ml\/templates\/(vivado|quartus)\/(ap_types|ac_types)\/|^test/pyte
 
 repos:
 - repo: https://github.com/psf/black
-  rev: 23.7.0
+  rev: 23.9.1
   hooks:
   - id: black
     language_version: python3
@@ -30,13 +30,13 @@ repos:
     args: ["--profile", "black", --line-length=125]
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.10.1
+  rev: v3.14.0
   hooks:
   - id: pyupgrade
     args: ["--py36-plus"]
 
 - repo: https://github.com/asottile/setup-cfg-fmt
-  rev: v2.4.0
+  rev: v2.5.0
   hooks:
   - id: setup-cfg-fmt
 

README.md

@@ -135,3 +135,8 @@ binary/ternary networks:
     year = "2021"
 }
 ```
+
+# Acknowledgments
+If you benefited from participating in our community, we ask that you please acknowledge the Fast Machine Learning collaboration, and particular individuals who helped you, in any publications.
+Please use the following text for this acknowledgment:
+  > We acknowledge the Fast Machine Learning collective as an open community of multi-domain experts and collaborators. This community and \<names of individuals\>, in particular, were important for the development of this project.

docs/reference.rst

@@ -86,6 +86,13 @@ binary/ternary networks:
         year = "2021"
     }
 
+Acknowledgments
+===============
+If you benefited from participating in our community, we ask that you please acknowledge the Fast Machine Learning collaboration, and particular individuals who helped you, in any publications.
+Please use the following text for this acknowledgment:
+  We acknowledge the Fast Machine Learning collective as an open community of multi-domain experts and collaborators. This community and \<names of individuals\>, in particular, were important for the development of this project.
+
+
 Contributors
 ============
 

hls4ml/backends/vivado/passes/convolution_templates.py

@@ -244,10 +244,12 @@ def __init__(self):
 }};\n"""
 
 sepconv1d_function_template = (
-    'nnet::separable_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {d}, {p}, {z}, {b});'
+    'nnet::separable_conv_1d_{data_format}<{input_t}, {dw_output_t}, {output_t}, {config}>('
+    '{input}, {output}, {d}, {p}, {z}, {b});'
 )
 sepconv2d_function_template = (
-    'nnet::separable_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {d}, {p}, {z}, {b});'
+    'nnet::separable_conv_2d_{data_format}<{input_t}, {dw_output_t}, {output_t}, {config}>('
+    '{input}, {output}, {d}, {p}, {z}, {b});'
 )
 
 sepconv1d_include_list = ['nnet_utils/nnet_conv1d.h', 'nnet_utils/nnet_sepconv1d_stream.h']
@@ -273,14 +275,17 @@ def format(self, node):
 
         # Depthwise config
         params = self._default_config_params(node)
+        # Override bias and bias_t since these are zeros in depthwise step of SepConv1D
+        params['bias'] = params['zero_bias']
+        params['bias_t'] = params['zero_bias_t']
         params['n_filt'] = params['n_chan']  # In depthwise step n_chan == n_filt
         params['dilation'] = node.get_attr('dilation', 1)
         params['nzeros'] = node.get_weights('depthwise').nzeros
         params['index'] = str(node.index) + '_depthwise'
         params['weight_t'] = node.get_weights('depthwise').type
         params['fill_fn'] = 'FillConv1DBuffer'
 
-        if node.get_attr("unscaled"):
+        if node.get_attr('unscaled'):
             params['scale_index_type'] = 'scale_index_unscaled'
         else:
             params['scale_index_type'] = 'scale_index_regular'
@@ -301,14 +306,11 @@ def format(self, node):
         depthwise_mult_config = self.depthwise_mult_template.format(**mult_params)
 
         # Pointwise config
-        params = self._default_config_params()
-        input_shape = self.get_input_variable().shape
-        if self.get_attr('data_format') == 'channels_last':
-            params['in_width'] = '*'.join([str(k) for k in input_shape[:-1]])
-            params['n_chan'] = input_shape[-1]
+        params = self._default_config_params(node)
+        if node.get_attr('data_format') == 'channels_last':
+            params['in_width'] = node.get_output_variable().shape[0]
         else:
-            params['in_width'] = '*'.join([str(k) for k in input_shape[1:]])
-            params['n_chan'] = input_shape[0]
+            params['in_width'] = node.get_output_variable().shape[1]
 
         params['filt_width'] = 1
         params['stride_width'] = 1
@@ -320,7 +322,7 @@ def format(self, node):
         params['instructions'] = '0'
         params['fill_fn'] = 'FillConv1DBuffer'
 
-        if node.get_attr("unscaled"):
+        if node.get_attr('unscaled'):
             params['scale_index_type'] = 'scale_index_unscaled'
         else:
             params['scale_index_type'] = 'scale_index_regular'
@@ -360,6 +362,7 @@ def __init__(self):
 
     def format(self, node):
         params = self._default_function_params(node)
+        params['dw_output_t'] = node.get_attr('dw_output_t').name
         params['data_format'] = 'cf' if node.get_attr('data_format') == 'channels_first' else 'cl'
         params['d'] = node.get_weights('depthwise').name
         params['p'] = node.get_weights('pointwise').name
@@ -398,12 +401,12 @@ def format(self, node):
         params['weight_t'] = node.get_weights('depthwise').type
         params['fill_fn'] = 'FillConv2DBuffer'
 
-        if node.get_attr("unscaled_h"):
+        if node.get_attr('unscaled_h'):
             params['scale_index_height_type'] = 'scale_index_unscaled'
         else:
             params['scale_index_height_type'] = 'scale_index_regular'
 
-        if node.get_attr("unscaled_w"):
+        if node.get_attr('unscaled_w'):
             params['scale_index_width_type'] = 'scale_index_unscaled'
         else:
             params['scale_index_width_type'] = 'scale_index_regular'
@@ -443,12 +446,12 @@ def format(self, node):
         params['instructions'] = '0'
         params['fill_fn'] = 'FillConv2DBuffer'
 
-        if node.get_attr("unscaled_h"):
+        if node.get_attr('unscaled_h'):
             params['scale_index_height_type'] = 'scale_index_unscaled'
         else:
             params['scale_index_height_type'] = 'scale_index_regular'
 
-        if node.get_attr("unscaled_w"):
+        if node.get_attr('unscaled_w'):
             params['scale_index_width_type'] = 'scale_index_unscaled'
         else:
             params['scale_index_width_type'] = 'scale_index_regular'
@@ -487,6 +490,7 @@ def __init__(self):
 
     def format(self, node):
         params = self._default_function_params(node)
+        params['dw_output_t'] = node.get_attr('dw_output_t').name
         params['data_format'] = 'cf' if node.get_attr('data_format') == 'channels_first' else 'cl'
         params['d'] = node.get_weights('depthwise').name
         params['p'] = node.get_weights('pointwise').name

hls4ml/backends/vivado/vivado_backend.py

@@ -28,7 +28,7 @@
     Softmax,
 )
 from hls4ml.model.optimizer import get_backend_passes, layer_optimizer
-from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType
+from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType, PackedType
 from hls4ml.report import parse_vivado_report
 from hls4ml.utils.fixed_point_utils import ceil_log2
 
@@ -75,6 +75,12 @@ def _register_layer_attributes(self):
             attrs.append(ChoiceAttribute('conv_implementation', choices=['LineBuffer', 'Encoded'], default='LineBuffer'))
             self.attribute_map[layer] = attrs
 
+        sep_conv_layers = [SeparableConv1D, SeparableConv2D]
+        for layer in sep_conv_layers:
+            attrs = self.attribute_map.get(layer, [])
+            attrs.append(TypeAttribute('dw_output', default=FixedPrecisionType(18, 8)))
+            self.attribute_map[layer] = attrs
+
     def _register_flows(self):
         initializers = self._get_layer_initializers()
         init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
@@ -288,6 +294,15 @@ def init_sepconv1d(self, layer):
         )  # TODO Once we have SeparableConv implementation for io_parallel this should be set properly
         layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
 
+        # Set the output type of the depthwise phase
+        dw_out_precision, _ = layer.model.config.get_precision(layer, 'dw_output')
+        dw_out_name = layer.name + '_dw_out_t'
+        if layer.model.config.get_config_value('IOType') == 'io_stream':
+            dw_output_t = PackedType(dw_out_name, dw_out_precision, layer.get_attr('n_chan'), n_pack=1)
+        else:
+            dw_output_t = NamedType(dw_out_name, dw_out_precision)
+        layer.set_attr('dw_output_t', dw_output_t)
+
     @layer_optimizer(Conv2D)
     def init_conv2d(self, layer):
         if len(layer.weights['weight'].data.shape) == 2:  # This can happen if we assign weights of Dense layer to 1x1 Conv2D
@@ -334,6 +349,15 @@ def init_sepconv2d(self, layer):
         )  # TODO Once we have SeparableConv implementation for io_parallel this should be set properly
         layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
 
+        # Set the output type of the depthwise phase
+        dw_out_precision, _ = layer.model.config.get_precision(layer, 'dw_output')
+        dw_out_name = layer.name + '_dw_out_t'
+        if layer.model.config.get_config_value('IOType') == 'io_stream':
+            dw_output_t = PackedType(dw_out_name, dw_out_precision, layer.get_attr('n_chan'), n_pack=1)
+        else:
+            dw_output_t = NamedType(dw_out_name, dw_out_precision)
+        layer.set_attr('dw_output_t', dw_output_t)
+
     @layer_optimizer(DepthwiseConv2D)
     def init_depconv2d(self, layer):
         if layer.model.config.is_resource_strategy(layer):

hls4ml/converters/keras/qkeras.py

@@ -54,6 +54,27 @@ def parse_qdepthwiseqconv_layer(keras_layer, input_names, input_shapes, data_rea
     layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
 
     layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
+
+    if keras_layer['config']['bias_quantizer'] is not None:
+        layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
+    else:
+        layer['bias_quantizer'] = None
+
+    return layer, output_shape
+
+
+@keras_handler('QSeparableConv1D', 'QSeparableConv2D')
+def parse_qsepconv_layer(keras_layer, input_names, input_shapes, data_reader):
+    assert 'QSeparableConv' in keras_layer['class_name']
+
+    if '1D' in keras_layer['class_name']:
+        layer, output_shape = parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader)
+    elif '2D' in keras_layer['class_name']:
+        layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
+
+    layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
+    layer['pointwise_quantizer'] = get_quantizer_from_config(keras_layer, 'pointwise')
+
     if keras_layer['config']['bias_quantizer'] is not None:
         layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
     else:

hls4ml/model/layers.py

@@ -56,7 +56,6 @@ class Layer:
         ConfigurableAttribute('trace', default=False),
         TypeAttribute('result'),
     ]
-    """"""
 
     @classproperty
     def expected_attributes(cls):
@@ -1343,8 +1342,10 @@ def initialize(self):
     'QConv2D': Conv2D,
     'QConv2DBatchnorm': Conv2DBatchnorm,
     'SeparableConv1D': SeparableConv1D,
+    'QSeparableConv1D': SeparableConv1D,
     'DepthwiseConv1D': DepthwiseConv1D,
     'SeparableConv2D': SeparableConv2D,
+    'QSeparableConv2D': SeparableConv2D,
     'DepthwiseConv2D': DepthwiseConv2D,
     'QDepthwiseConv2D': DepthwiseConv2D,
     'BatchNormalization': BatchNormalization,

hls4ml/model/profiling.py

@@ -343,21 +343,22 @@ def activations_keras(model, X, fmt='longform', plot='boxplot'):
         # return summary statistics for matplotlib.axes.Axes.bxp
         # or histogram bin edges and heights
         data = []
-
-    for layer in model.layers:
-        print(f"   {layer.name}")
-        if not isinstance(layer, keras.layers.InputLayer):
-            y = _get_output(layer, X, model.input).flatten()
-            y = abs(y[y != 0])
-            if len(y) == 0:
-                print(f'Activations for {layer.name} are only zeros, ignoring.')
-                continue
-            if fmt == 'longform':
-                data['x'].extend(y.tolist())
-                data['weight'].extend([layer.name for i in range(len(y))])
-            elif fmt == 'summary':
-                data.append(array_to_summary(y, fmt=plot))
-                data[-1]['weight'] = layer.name
+    outputs = _get_outputs(
+        [layer for layer in model.layers if not isinstance(layer, keras.layers.InputLayer)], X, model.input
+    )
+    for layer_name, y in outputs.items():
+        print(f"   {layer_name}")
+        y = y.flatten()
+        y = abs(y[y != 0])
+        if len(y) == 0:
+            print(f'Activations for {layer_name} are only zeros, ignoring.')
+            continue
+        if fmt == 'longform':
+            data['x'].extend(y.tolist())
+            data['weight'].extend([layer_name for i in range(len(y))])
+        elif fmt == 'summary':
+            data.append(array_to_summary(y, fmt=plot))
+            data[-1]['weight'] = layer_name
 
     if fmt == 'longform':
         data = pandas.DataFrame(data)
@@ -544,10 +545,10 @@ def _is_ignored_layer(layer):
     return False
 
 
-def _get_output(layer, X, model_input):
-    """Get output of partial model"""
-    partial_model = keras.models.Model(inputs=model_input, outputs=layer.output)
-    y = partial_model.predict(X)
+def _get_outputs(layers, X, model_input):
+    """Get outputs of intermediate layers"""
+    partial_models = keras.models.Model(inputs=model_input, outputs=[layer.output for layer in layers])
+    y = partial_models.predict(X)
     return y
 
 
@@ -562,37 +563,30 @@ def get_ymodel_keras(keras_model, X):
     Returns:
         dict: A dictionary in the form {"layer_name": ouput array of layer}.
     """
-
     ymodel = {}
-
+    traced_layers = []
+    layer_names = []
     for layer in keras_model.layers:
-        print(f"Processing {layer.name} in Keras model...")
-        if not _is_ignored_layer(layer):
-            # If the layer has activation integrated then separate them
-            # Note that if the layer is a standalone activation layer then skip this
-            if hasattr(layer, 'activation') and not (
-                isinstance(layer, keras.layers.Activation) or isinstance(layer, qkeras.qlayers.QActivation)
-            ):
-                if layer.activation:
-                    if layer.activation.__class__.__name__ == "linear":
-                        ymodel[layer.name] = _get_output(layer, X, keras_model.input)
-
-                    else:
-                        temp_activation = layer.activation
-                        layer.activation = None
-                        # Get output for layer without activation
-                        ymodel[layer.name] = _get_output(layer, X, keras_model.input)
-
-                        # Add the activation back
-                        layer.activation = temp_activation
-                        # Get ouput for activation
-                        ymodel[layer.name + f"_{temp_activation.__class__.__name__}"] = _get_output(
-                            layer, X, keras_model.input
-                        )
-                else:
-                    ymodel[layer.name] = _get_output(layer, X, keras_model.input)
-            else:
-                ymodel[layer.name] = _get_output(layer, X, keras_model.input)
+        if _is_ignored_layer(layer):
+            continue
+        # If the layer has activation integrated then separate them
+        # Note that if the layer is a standalone activation layer then skip this
+        name = layer.name
+        if (
+            hasattr(layer, "activation")
+            and layer.activation.__name__ != "linear"
+            and not isinstance(layer, (keras.layers.Activation, qkeras.qlayers.QActivation))
+        ):
+            tmp_activation = layer.activation
+            layer.activation = None
+            ymodel.update({layer.name: _get_outputs([layer], X, keras_model.input)})
+            layer.activation = tmp_activation
+            name = layer.name + f"_{tmp_activation.__name__}"
+        traced_layers.append(layer)
+        layer_names.append(name)
+    outputs = _get_outputs(traced_layers, X, keras_model.input)
+    for name, output in zip(layer_names, outputs):
+        ymodel[name] = output
     print("Done taking outputs for Keras model.")
     return ymodel
 

hls4ml/model/types.py

@@ -564,7 +564,7 @@ def update_precision(self, new_precision):
                 # to right of decimal point
                 lsb = 2**-new_precision.fractional
                 decimal_spaces = len(str(lsb).split('.')[1])
-                self.precision_fmt = f'{{:{decimal_spaces}f}}'
+                self.precision_fmt = f'{{:.{decimal_spaces}f}}'
             else:
                 self.precision_fmt = '{:.0f}'
         else: