BERT NER Demo: support dynamic shape (#3337) (#3344)

* BERT NER Demo: support dynamic shape

* Update demos/bert_named_entity_recognition_demo/python/bert_named_entity_recognition_demo.py

Co-authored-by: Aleksei Korobeinikov <[email protected]>

* Update demos/common/python/openvino/model_zoo/model_api/adapters/openvino_adapter.py

Co-authored-by: Aleksei Korobeinikov <[email protected]>

* Apply suggestions from code review

Co-authored-by: Anzhella Pankratova <[email protected]>

* Apply suggestions from code review

Co-authored-by: Anzhella Pankratova <[email protected]>

* fix bounds for filtered_labels_id in BertNamedEntityRecognition wrapper (#3341)

Co-authored-by: Aleksei Korobeinikov <[email protected]>
Co-authored-by: Anzhella Pankratova <[email protected]>

Co-authored-by: Aleksei Korobeinikov <[email protected]>
Co-authored-by: Anzhella Pankratova <[email protected]>

Author: 3 people

demos/bert_named_entity_recognition_demo/python/README.md

@@ -70,6 +70,8 @@ Options:
   -nthreads NUM_THREADS, --num_threads NUM_THREADS
                         Optional. Number of threads to use for inference on
                         CPU (including HETERO cases).
+  --dynamic_shape       Optional. Run model with dynamic input sequence. If
+                        not provided, input sequence is padded to max_seq_len
 ```
 
 ## Demo Inputs
@@ -118,7 +120,7 @@ Exemplary command:
 ## Classifying Documents with Long Texts
 
 Notice that when the original "context" (text from the url) does not fit the model input
-(128 for the Bert-Base), the demo reshapes model to maximum sentence length in the "context".
+(128 for the Bert-Base), the demo reshapes model to maximum sentence length in the "context" and pad all input sequences to maximum sentence length if model executed with static shape.
 
 ## See Also
 

demos/bert_named_entity_recognition_demo/python/bert_named_entity_recognition_demo.py

@@ -72,6 +72,8 @@ def build_argparser():
                       default='', type=str)
     args.add_argument('-nthreads', '--num_threads', default=None, type=int,
                       help='Optional. Number of threads to use for inference on CPU (including HETERO cases).')
+    args.add_argument('--dynamic_shape', action='store_true',
+                      help='Optional. Run model with dynamic input sequence. If not provided, input sequence is padded to max_seq_len')
     return parser
 
 
@@ -116,9 +118,10 @@ def main():
     elif args.adapter == 'ovms':
         model_adapter = OVMSAdapter(args.model)
 
-    model = BertNamedEntityRecognition(model_adapter, {'vocab': vocab, 'input_names': args.input_names})
+    enable_padding = not args.dynamic_shape
+    model = BertNamedEntityRecognition(model_adapter, {'vocab': vocab, 'input_names': args.input_names, 'enable_padding': enable_padding})
     if max_sentence_length > model.max_length:
-        model.reshape(max_sentence_length)
+        model.reshape(max_sentence_length if enable_padding else (1, max_sentence_length))
     model.log_layers_info()
 
     pipeline = AsyncPipeline(model)

demos/common/python/openvino/model_zoo/model_api/adapters/openvino_adapter.py

@@ -18,7 +18,7 @@
 from pathlib import Path
 
 try:
-    from openvino.runtime import AsyncInferQueue, Core, PartialShape, layout_helpers, get_version
+    from openvino.runtime import AsyncInferQueue, Core, PartialShape, layout_helpers, get_version, Dimension
     openvino_absent = False
 except ImportError:
     openvino_absent = True
@@ -89,20 +89,21 @@ def log_runtime_settings(self):
     def get_input_layers(self):
         inputs = {}
         for input in self.model.inputs:
-            input_layout = self.get_layout_for_input(input)
-            inputs[input.get_any_name()] = Metadata(input.get_names(), list(input.shape), input_layout, input.get_element_type().get_type_name())
+            input_shape = get_input_shape(input)
+            input_layout = self.get_layout_for_input(input, input_shape)
+            inputs[input.get_any_name()] = Metadata(input.get_names(), input_shape, input_layout, input.get_element_type().get_type_name())
         inputs = self._get_meta_from_ngraph(inputs)
         return inputs
 
-    def get_layout_for_input(self, input) -> str:
+    def get_layout_for_input(self, input, shape=None) -> str:
         input_layout = ''
         if self.model_parameters['input_layouts']:
             input_layout = Layout.from_user_layouts(input.get_names(), self.model_parameters['input_layouts'])
         if not input_layout:
             if not layout_helpers.get_layout(input).empty:
                 input_layout = Layout.from_openvino(input)
             else:
-                input_layout = Layout.from_shape(input.shape)
+                input_layout = Layout.from_shape(shape if shape is not None else input.shape)
         return input_layout
 
     def get_output_layers(self):
@@ -114,7 +115,9 @@ def get_output_layers(self):
         return outputs
 
     def reshape_model(self, new_shape):
-        new_shape = {k: PartialShape(v) for k, v in new_shape.items()}
+        new_shape = {name: PartialShape(
+            [Dimension(dim) if not isinstance(dim, tuple) else Dimension(dim[0], dim[1])
+            for dim in shape]) for name, shape in new_shape.items()}
         self.model.reshape(new_shape)
 
     def get_raw_result(self, request):
@@ -157,3 +160,24 @@ def operations_by_type(self, operation_type):
                 layer_name = node.get_friendly_name()
                 layers_info[layer_name] = Metadata(type=node.get_type_name(), meta=node.get_attributes())
         return layers_info
+
+
+def get_input_shape(input_tensor):
+    def string_to_tuple(string, casting_type=int):
+        processed = string.replace(' ', '').replace('(', '').replace(')', '').split(',')
+        processed = filter(lambda x: x, processed)
+        return tuple(map(casting_type, processed)) if casting_type else tuple(processed)
+    if not input_tensor.partial_shape.is_dynamic:
+        return list(input_tensor.shape)
+    ps = str(input_tensor.partial_shape)
+    preprocessed = ps.replace('{', '(').replace('}', ')').replace('?', '-1')
+    preprocessed = preprocessed.replace('(', '').replace(')', '')
+    if '..' in preprocessed:
+        shape_list = []
+        for dim in preprocessed.split(','):
+            if '..' in dim:
+                shape_list.append(string_to_tuple(dim.replace('..', ',')))
+            else:
+                shape_list.append(int(dim))
+        return shape_list
+    return string_to_tuple(preprocessed)

demos/common/python/openvino/model_zoo/model_api/models/bert.py

@@ -14,7 +14,7 @@
 import numpy as np
 
 from .model import Model, WrapperError
-from .types import DictValue, NumericalValue, StringValue
+from .types import DictValue, NumericalValue, StringValue, BooleanValue
 
 
 class Bert(Model):
@@ -37,13 +37,15 @@ def parameters(cls):
         parameters.update({
             'vocab': DictValue(),
             'input_names': StringValue(description='Comma-separated names of input layers'),
+            'enable_padding': BooleanValue(
+                description='Should be input sequence padded to max sequence len or not', default_value=True
+            )
         })
         return parameters
 
     def preprocess(self, inputs):
         input_ids, attention_mask, token_type_ids = self.form_request(inputs)
-
-        pad_len = self.pad_input(input_ids, attention_mask, token_type_ids)
+        pad_len = self.pad_input(input_ids, attention_mask, token_type_ids) if self.enable_padding else 0
         meta = {'pad_len': pad_len, 'inputs': inputs}
 
         return self.create_input_dict(input_ids, attention_mask, token_type_ids), meta
@@ -79,7 +81,7 @@ def reshape(self, new_length):
         default_input_shape = input_info.shape
         super().reshape(new_shapes)
         self.logger.debug("\tReshape model from {} to {}".format(default_input_shape, new_shapes[input_name]))
-        self.max_length = new_length
+        self.max_length = new_length if not isinstance(new_length, tuple) else new_length[1]
 
 
 class BertNamedEntityRecognition(Bert):

demos/common/python/openvino/model_zoo/model_api/models/types.py

@@ -34,7 +34,7 @@ def validate(self, value):
     def get_value(self, value):
         errors = self.validate(value)
         if len(errors) == 0:
-            return value if value else self.default_value
+            return value if value is not None else self.default_value
 
     def build_error():
         pass

models/public/bert-base-ner/model.yml

@@ -86,6 +86,7 @@ conversion_to_onnx_args:
   - --input-shapes=[1,128],[1,128],[1,128]
   - --output-file=$conv_dir/bert-base-ner.onnx
   - --inputs-dtype=long
+  - '--conversion-param=dynamic_axes={"input_ids": {0: "batch_size", 1: "sequence_len"},"attention_mask": {0: "batch_size", 1: "sequence_len"}, "token_type_ids": {0: "batch_size", 1: "sequence_len"}, "output": {0: "batch_size", 1: "sequence_len"}}'
 input_info:
   - name: input_ids
     shape: [1, 128]