Merge branch 'main' into hw_opt_p2

Author: vloncar

docs/api/configuration.rst

@@ -9,15 +9,18 @@ We currently support two ways of setting hls4ml's model configuration. This page
 
 .. contents:: \
 
+The Python API approach is recommended for most users as there are more utilities to help create the configuration dictionaries.
 
 **NOTE:**
 
 
 *
   One important part of ``hls4ml`` to remember is that the user is responsible for the format of the inputs.  There is no automatic formatting or normalization so this must be done in the training.
 
-*
+..
+  *
   For developers, you might also want to checkout this section: `Detailed configuration in converted hls codes <#detailed-configuration-in-converted-hls-codes>`_.
+  *Broken link*
 
 ----
 
@@ -31,11 +34,26 @@ Using hls4ml, you can quickly generate a simple configuration dictionary from a
    import hls4ml
    config = hls4ml.utils.config_from_keras_model(model, granularity='model')
 
-For more advanced and detailed configuration, you can also set them through the created dictionary. For example, to change the reuse factor:
+This python dictionary can be edited as needed. A more advanced configuration can be generated by, for example:
+
+.. code-block:: python
+
+   import hls4ml
+   config = hls4ml.utils.config_from_keras_model(
+        model,
+        granularity='name',
+        default_precision='fixed<16,6>',
+        backend='Vitis')
+
+This will include per-layer configuration based on the model. Including the backend is recommended because some configation options depend on the backend. Note, the precisions at the
+higher granularites usually default to 'auto', which means that ``hls4ml`` will try to set it automatically. Note that higher granularity settings take precendence
+over model-level settings. See :py:class:`~hls4ml.utils.config.config_from_keras_model` for more information on the various options.
+
+One can override specific values before using the configuration:
 
 .. code-block:: python
 
-   config['Model']['ReuseFactor'] = 2
+   config['LayerName']['fc1']['ReuseFactor'] = 2
 
 Or to set the precision of a specific layer's weight:
 
@@ -45,6 +63,20 @@ Or to set the precision of a specific layer's weight:
 
 To better understand how the configuration hierachy works, refer to the next section for more details.
 
+Finally, one then uses the configuration to create an hls model:
+
+.. code-block:: python
+
+  hls_model = hls4ml.converters.convert_from_keras_model(
+        model,
+        hls_config=config,
+        output_dir="my_project_dir",
+        io_type='io_stream',
+        backend='Vitis'
+    )
+
+See :py:class:`~hls4ml.converters.convert_from_keras_model` for more information on the various options.
+
 ----
 
 2. YAML Configuration file

docs/setup.rst

@@ -57,7 +57,7 @@ To run FPGA synthesis, installation of following tools is required:
 
 * Xilinx Vivado HLS 2018.2 to 2020.1 for synthesis for Xilinx FPGAs
 
-  * Vitis HLS 2022.1 or newer is required for synthesis for Xilinx FPGAs using the experimental ``Vitis`` backend.
+  * Vitis HLS 2022.2 or newer is required for synthesis for Xilinx FPGAs using the ``Vitis`` backend.
 
 * Intel Quartus 20.1 to 21.4 for the synthesis for Intel FPGAs
 

docs/status.rst

@@ -81,7 +81,7 @@ Other feature notes:
 * ``hls4ml`` is tested on Linux, and supports
    * Vivado HLS versions 2018.2 to 2020.1
    * Intel HLS versions 20.1 to 21.4
-   * Vitis HLS versions 2020.2 to 2022.2 (experimentally)
+   * Vitis HLS versions 2022.2 to 2024.1
 * Windows and macOS are not supported
 * BDT support has moved to the `Conifer <https://github.com/thesps/conifer>`__ package
 

hls4ml/backends/catapult/catapult_backend.py

@@ -1,5 +1,6 @@
 import os
 import sys
+from warnings import warn
 
 import numpy as np
 
@@ -110,6 +111,8 @@ def _register_flows(self):
             'catapult:inplace_stream_flatten',
             'catapult:skip_softmax',
             'catapult:fix_softmax_table_size',
+            'catapult:process_fixed_point_quantizer_layer',
+            'infer_precision_types',
         ]
         optimization_flow = register_flow('optimize', optimization_passes, requires=[init_flow], backend=self.name)
 
@@ -119,6 +122,7 @@ def _register_flows(self):
             'catapult:generate_conv_streaming_instructions',
             'catapult:apply_resource_strategy',
             'catapult:generate_conv_im2col',
+            'catapult:apply_winograd_kernel_transformation',
         ]
         catapult_types_flow = register_flow('specific_types', catapult_types, requires=[init_flow], backend=self.name)
 
@@ -152,9 +156,8 @@ def _register_flows(self):
         ]
 
         if len(extras) > 0:
-            extras_flow = register_flow('extras', extras, requires=[init_flow], backend=self.name)
-        else:
-            extras_flow = None
+            for opt in extras:
+                warn(f'WARNING: Optimizer "{opt}" is not part of any flow and will not be executed.')
 
         ip_flow_requirements = [
             'optimize',
@@ -163,10 +166,8 @@ def _register_flows(self):
             quantization_flow,
             optimization_flow,
             catapult_types_flow,
-            extras_flow,
             template_flow,
         ]
-        ip_flow_requirements = list(filter(None, ip_flow_requirements))
 
         self._default_flow = register_flow('ip', None, requires=ip_flow_requirements, backend=self.name)
 

hls4ml/backends/catapult/passes/conv_same_pad.py

@@ -6,10 +6,8 @@ class InsertZeroPaddingBeforeConv1D(OptimizerPass):
     name = 'insert_zero_padding_before_conv1d'
 
     def match(self, node):
-        is_match = (
-            isinstance(node, (Conv1D, SeparableConv1D))
-            and ((node.get_attr('padding') == 'same') or (node.get_attr('padding') == 'causal'))
-            and node.get_attr('filt_width') != 1
+        is_match = isinstance(node, (Conv1D, SeparableConv1D)) and (
+            (node.get_attr('pad_left') != 0) or (node.get_attr('pad_right') != 0)
         )
         return is_match
 
@@ -37,7 +35,6 @@ def transform(self, model, node):
         }
 
         # Switch Conv1D layer padding to 'valid'
-        node.set_attr('padding', 'valid')
         node.set_attr('pad_left', 0)
         node.set_attr('pad_right', 0)
         node.set_attr('in_width', out_width)
@@ -54,11 +51,11 @@ class InsertZeroPaddingBeforeConv2D(OptimizerPass):
     name = 'insert_zero_padding_before_conv2d'
 
     def match(self, node):
-        is_match = (
-            isinstance(node, (Conv2D, SeparableConv2D))
-            and node.get_attr('padding') == 'same'
-            and node.get_attr('filt_height') != 1
-            and node.get_attr('filt_width') != 1
+        is_match = isinstance(node, (Conv2D, SeparableConv2D)) and (
+            (node.get_attr('pad_left') != 0)
+            or (node.get_attr('pad_right') != 0)
+            or (node.get_attr('pad_top') != 0)
+            or (node.get_attr('pad_bottom') != 0)
         )
         return is_match
 
@@ -93,7 +90,6 @@ def transform(self, model, node):
         }
 
         # Switch Conv2D layer padding to 'valid'
-        node.set_attr('padding', 'valid')
         node.set_attr('pad_top', 0)
         node.set_attr('pad_bottom', 0)
         node.set_attr('pad_left', 0)

hls4ml/backends/quartus/quartus_backend.py

@@ -55,6 +55,7 @@ def _register_flows(self):
             'quartus:transform_types',
             'quartus:register_bram_weights',
             'quartus:apply_resource_strategy',
+            'quartus:generate_conv_im2col',
             'quartus:apply_winograd_kernel_transformation',
         ]
         quartus_types_flow = register_flow('specific_types', quartus_types, requires=[init_flow], backend=self.name)
@@ -103,9 +104,8 @@ def _register_flows(self):
         ]
 
         if len(extras) > 0:
-            extras_flow = register_flow('extras', extras, requires=[init_flow], backend=self.name)
-        else:
-            extras_flow = None
+            for opt in extras:
+                warn(f'WARNING: Optimizer "{opt}" is not part of any flow and will not be executed.')
 
         ip_flow_requirements = [
             'optimize',
@@ -114,10 +114,8 @@ def _register_flows(self):
             quantization_flow,
             optimization_flow,
             quartus_types_flow,
-            extras_flow,
             template_flow,
         ]
-        ip_flow_requirements = list(filter(None, ip_flow_requirements))
 
         self._default_flow = register_flow('ip', None, requires=ip_flow_requirements, backend=self.name)
 

hls4ml/backends/vivado/passes/conv_same_pad.py

@@ -6,10 +6,8 @@ class InsertZeroPaddingBeforeConv1D(OptimizerPass):
     name = 'insert_zero_padding_before_conv1d'
 
     def match(self, node):
-        is_match = (
-            isinstance(node, (Conv1D, SeparableConv1D))
-            and ((node.get_attr('padding') == 'same') or (node.get_attr('padding') == 'causal'))
-            and node.get_attr('filt_width') != 1
+        is_match = isinstance(node, (Conv1D, SeparableConv1D)) and (
+            (node.get_attr('pad_left') != 0) or (node.get_attr('pad_right') != 0)
         )
         return is_match
 
@@ -37,7 +35,6 @@ def transform(self, model, node):
         }
 
         # Switch Conv1D layer padding to 'valid'
-        node.set_attr('padding', 'valid')
         node.set_attr('pad_left', 0)
         node.set_attr('pad_right', 0)
         node.set_attr('in_width', out_width)
@@ -54,11 +51,11 @@ class InsertZeroPaddingBeforeConv2D(OptimizerPass):
     name = 'insert_zero_padding_before_conv2d'
 
     def match(self, node):
-        is_match = (
-            isinstance(node, (Conv2D, SeparableConv2D))
-            and node.get_attr('padding') == 'same'
-            and node.get_attr('filt_height') != 1
-            and node.get_attr('filt_width') != 1
+        is_match = isinstance(node, (Conv2D, SeparableConv2D)) and (
+            (node.get_attr('pad_left') != 0)
+            or (node.get_attr('pad_right') != 0)
+            or (node.get_attr('pad_top') != 0)
+            or (node.get_attr('pad_bottom') != 0)
         )
         return is_match
 
@@ -93,7 +90,6 @@ def transform(self, model, node):
         }
 
         # Switch Conv2D layer padding to 'valid'
-        node.set_attr('padding', 'valid')
         node.set_attr('pad_top', 0)
         node.set_attr('pad_bottom', 0)
         node.set_attr('pad_left', 0)

hls4ml/backends/vivado/vivado_backend.py

@@ -148,9 +148,8 @@ def _register_flows(self):
         ]
 
         if len(extras) > 0:
-            extras_flow = register_flow('extras', extras, requires=[init_flow], backend=self.name)
-        else:
-            extras_flow = None
+            for opt in extras:
+                warn(f'WARNING: Optimizer "{opt}" is not part of any flow and will not be executed.')
 
         ip_flow_requirements = [
             'optimize',
@@ -159,10 +158,8 @@ def _register_flows(self):
             quantization_flow,
             optimization_flow,
             vivado_types_flow,
-            extras_flow,
             template_flow,
         ]
-        ip_flow_requirements = list(filter(None, ip_flow_requirements))
 
         self._default_flow = register_flow('ip', None, requires=ip_flow_requirements, backend=self.name)
 

hls4ml/converters/__init__.py

@@ -10,6 +10,8 @@
 from hls4ml.converters.keras_to_hls import get_supported_keras_layers  # noqa: F401
 from hls4ml.converters.keras_to_hls import parse_keras_model  # noqa: F401
 from hls4ml.converters.keras_to_hls import keras_to_hls, register_keras_layer_handler
+
+# from hls4ml.converters.pytorch_to_hls import parse_pytorch_model  # noqa: F401
 from hls4ml.model import ModelGraph
 from hls4ml.utils.config import create_config
 from hls4ml.utils.symbolic_utils import LUTFunction
@@ -238,7 +240,6 @@ def convert_from_keras_model(
 
 def convert_from_pytorch_model(
     model,
-    input_shape,
     output_dir='my-hls-test',
     project_name='myproject',
     input_data_tb=None,
@@ -251,7 +252,6 @@ def convert_from_pytorch_model(
 
     Args:
         model: PyTorch model to convert.
-        input_shape (list): The shape of the input tensor. First element is the batch size, needs to be None
         output_dir (str, optional): Output directory of the generated HLS project. Defaults to 'my-hls-test'.
         project_name (str, optional): Name of the HLS project. Defaults to 'myproject'.
         input_data_tb (str, optional): String representing the path of input data in .npy or .dat format that will be
@@ -293,17 +293,16 @@ def convert_from_pytorch_model(
     config = create_config(output_dir=output_dir, project_name=project_name, backend=backend, **kwargs)
 
     config['PytorchModel'] = model
-    config['InputShape'] = input_shape
     config['InputData'] = input_data_tb
     config['OutputPredictions'] = output_data_tb
     config['HLSConfig'] = {}
 
     if hls_config is None:
         hls_config = {}
 
-    model_config = hls_config.get('Model', None)
+    model_config = hls_config.get('Model')
     config['HLSConfig']['Model'] = _check_model_config(model_config)
-
+    config['InputShape'] = hls_config.get('InputShape')
     _check_hls_config(config, hls_config)
 
     return pytorch_to_hls(config)

hls4ml/converters/keras/convolution.py

@@ -30,10 +30,9 @@ def parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader):
         layer['n_filt'] = layer['n_chan'] * layer.get('depth_multiplier')
     layer['filt_width'] = keras_layer['config']['kernel_size'][0]
     layer['stride_width'] = keras_layer['config']['strides'][0]
-    layer['padding'] = keras_layer['config']['padding']
 
     (layer['out_width'], layer['pad_left'], layer['pad_right']) = compute_padding_1d(
-        layer['padding'], layer['in_width'], layer['stride_width'], layer['filt_width']
+        keras_layer['config']['padding'], layer['in_width'], layer['stride_width'], layer['filt_width']
     )
 
     if layer['data_format'] == 'channels_last':
@@ -74,7 +73,6 @@ def parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader):
     layer['filt_width'] = keras_layer['config']['kernel_size'][1]
     layer['stride_height'] = keras_layer['config']['strides'][0]
     layer['stride_width'] = keras_layer['config']['strides'][1]
-    layer['padding'] = keras_layer['config']['padding']
 
     (
         layer['out_height'],
@@ -84,7 +82,7 @@ def parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader):
         layer['pad_left'],
         layer['pad_right'],
     ) = compute_padding_2d(
-        layer['padding'],
+        keras_layer['config']['padding'],
         layer['in_height'],
         layer['in_width'],
         layer['stride_height'],