update interface for depth multiplier, though HLS doesn't yet implement it

Author: jmitrevs

hls4ml/backends/fpga/passes/codegen.py

@@ -6,6 +6,7 @@
 class GenerateConvIm2col(OptimizerPass):
     '''Generates tcode for im2col step of 1D/2d convolution'''
 
+    # Note, DepthwizeConv1D/2D also matches because it inherits from Conv1D/2D
     def match(self, node):
         return (
             isinstance(node, (Conv1D, Conv2D, SeparableConv1D, SeparableConv2D))

hls4ml/backends/vivado/vivado_backend.py

@@ -14,6 +14,7 @@
     Conv1D,
     Conv2D,
     Dense,
+    DepthwiseConv1D,
     DepthwiseConv2D,
     Embedding,
     GarNet,
@@ -314,6 +315,31 @@ def init_sepconv1d(self, layer):
             dw_output_t = NamedType(dw_out_name, dw_out_precision)
         layer.set_attr('dw_output_t', dw_output_t)
 
+    @layer_optimizer(DepthwiseConv1D)
+    def init_depconv1d(self, layer):
+        if layer.model.config.is_resource_strategy(layer):
+            layer.set_attr('strategy', 'resource')
+            n_in, n_out = self.get_layer_mult_size(layer)
+            self.set_closest_reuse_factor(layer, n_in, n_out)
+        else:
+            layer.set_attr('strategy', 'latency')
+
+        out_width = layer.get_output_variable().shape[0]
+        chosen_pf = layer.model.config.get_layer_config_value(layer, 'ParallelizationFactor', 1)
+        valid_pf = self.get_valid_conv_partition_splits(1, out_width)
+        if chosen_pf not in valid_pf:
+            closest_pf = self.get_closest_reuse_factor(valid_pf, chosen_pf)
+            valid_pf_str = ','.join(map(str, valid_pf))
+            print(
+                f'WARNING: Invalid ParallelizationFactor={chosen_pf} in layer "{layer.name}".'
+                f'Using ParallelizationFactor={closest_pf} instead. Valid ParallelizationFactor(s): {valid_pf_str}.'
+            )
+        else:
+            closest_pf = chosen_pf
+        layer.set_attr('n_partitions', out_width // closest_pf)
+
+        layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
+
     @layer_optimizer(Conv2D)
     def init_conv2d(self, layer):
         if len(layer.weights['weight'].data.shape) == 2:  # This can happen if we assign weights of Dense layer to 1x1 Conv2D

hls4ml/converters/keras/convolution.py

@@ -27,7 +27,7 @@ def parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader):
     if 'filters' in keras_layer['config']:
         layer['n_filt'] = keras_layer['config']['filters']
     else:
-        layer['n_filt'] = layer['n_chan']
+        layer['n_filt'] = layer['n_chan'] * layer.get('depth_multiplier')
     layer['filt_width'] = keras_layer['config']['kernel_size'][0]
     layer['stride_width'] = keras_layer['config']['strides'][0]
     layer['padding'] = keras_layer['config']['padding']
@@ -69,7 +69,7 @@ def parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader):
     if 'filters' in keras_layer['config']:
         layer['n_filt'] = keras_layer['config']['filters']
     else:
-        layer['n_filt'] = layer['n_chan']
+        layer['n_filt'] = layer['n_chan'] * layer.get('depth_multiplier')
     layer['filt_height'] = keras_layer['config']['kernel_size'][0]
     layer['filt_width'] = keras_layer['config']['kernel_size'][1]
     layer['stride_height'] = keras_layer['config']['strides'][0]

hls4ml/model/layers.py

@@ -489,6 +489,7 @@ class DepthwiseConv1D(Conv1D):
         Attribute('out_width'),
         Attribute('n_chan'),
         Attribute('depth_multiplier', default=1),
+        Attribute('n_filt'),  # = n_chan * depth_multiplier
         Attribute('filt_width'),
         Attribute('stride_width'),
         Attribute('pad_left'),
@@ -501,10 +502,10 @@ class DepthwiseConv1D(Conv1D):
 
     def initialize(self):
         if self.get_attr('data_format') == 'channels_last':
-            shape = [self.attributes['out_width'], self.attributes['n_chan'] * self.attributes['depth_multiplier']]
+            shape = [self.attributes['out_width'], self.attributes['n_filt']]
             dims = [f'OUT_HEIGHT_{self.index}', f'N_CHAN_{self.index}']
         else:
-            shape = [self.attributes['n_chan'] * self.attributes['depth_multiplier'], self.attributes['out_width']]
+            shape = [self.attributes['n_filt'], self.attributes['out_width']]
             dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}']
         self.add_output_variable(shape, dims)
 
@@ -513,7 +514,6 @@ def initialize(self):
         )
 
         self.add_bias(quantizer=self.get_attr('bias_quantizer'))
-        self.set_attr('n_filt', self.get_attr('n_chan') * self.get_attr('depth_multiplier'))
 
 
 class Conv2D(Layer):
@@ -658,6 +658,7 @@ class DepthwiseConv2D(Conv2D):
         Attribute('out_width'),
         Attribute('n_chan'),
         Attribute('depth_multiplier', default=1),
+        Attribute('n_filt'),  # = n_chan * depth_multiplier
         Attribute('filt_height'),
         Attribute('filt_width'),
         Attribute('stride_height'),
@@ -677,12 +678,12 @@ def initialize(self):
             shape = [
                 self.attributes['out_height'],
                 self.attributes['out_width'],
-                self.attributes['n_chan'] * self.attributes['depth_multiplier'],
+                self.attributes['n_filt'],
             ]
             dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
         else:
             shape = [
-                self.attributes['n_chan'] * self.attributes['depth_multiplier'],
+                self.attributes['n_filt'],
                 self.attributes['out_height'],
                 self.attributes['out_width'],
             ]
@@ -694,7 +695,6 @@ def initialize(self):
         )
 
         self.add_bias(quantizer=self.get_attr('bias_quantizer'))
-        self.set_attr('n_filt', self.get_attr('n_chan') * self.get_attr('depth_multiplier'))
 
 
 class Pooling1D(Layer):

hls4ml/model/optimizer/passes/seperable_to_dw_conv.py

@@ -72,7 +72,7 @@ def transform(self, model, node):
 
         # creating the attributes
         dw_attributes = {k: node.attributes[k] for k in SeperableToDepthwiseAndConv._dw_attributes if k in node.attributes}
-
+        dw_attributes['n_filt'] = dw_attributes['n_chan'] * dw_attributes['depth_multiplier']
         dw_attributes['use_bias'] = False
 
         new_dw = model.make_node('DepthwiseConv' + dim, dw_name, dw_attributes, [node.inputs[0]])

hls4ml/templates/vivado/nnet_utils/nnet_sepconv1d_latency.h

@@ -10,7 +10,7 @@ namespace nnet {
 template <class data_T, class res_T, typename CONFIG_T>
 void depthwise_conv_1d_latency_cl(data_T data[CONFIG_T::in_width * CONFIG_T::n_chan],
                                   res_T res[CONFIG_T::out_width * CONFIG_T::n_filt],
-                                  typename CONFIG_T::weight_t weights[CONFIG_T::filt_width * CONFIG_T::n_chan],
+                                  typename CONFIG_T::weight_t weights[CONFIG_T::filt_width * CONFIG_T::n_filt],
                                   typename CONFIG_T::bias_t biases[CONFIG_T::n_filt]) {
 
     constexpr unsigned mult_n_in = CONFIG_T::filt_width * CONFIG_T::n_chan;

hls4ml/templates/vivado/nnet_utils/nnet_sepconv2d_latency.h

@@ -11,7 +11,7 @@ template <class data_T, class res_T, typename CONFIG_T>
 void depthwise_conv_2d_latency_cl(
     data_T data[CONFIG_T::in_height * CONFIG_T::in_width * CONFIG_T::n_chan],
     res_T res[CONFIG_T::out_height * CONFIG_T::out_width * CONFIG_T::n_filt],
-    typename CONFIG_T::weight_t weights[CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan],
+    typename CONFIG_T::weight_t weights[CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_filt],
     typename CONFIG_T::bias_t biases[CONFIG_T::n_filt]) {
 
     constexpr unsigned mult_n_in = CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan;