Introduce FrameworkAdapter (#2844)

* Add inference adapter

* Do not refer directly to model_executor attributes

* Move Openvino interface from the demo and AsyncPipeline to Adapter, implement RemoteAdapter, add documentation to ModelAdapter Abstract class

* Add the __call__ method to ModelBase class

* Refactoring

* Update get_input_layers(), get_output_layers(), add Metadata structure, separate the model reading from the loading

* Update model wrappers for object_detection_demo, except YOLO wrappers

* Update Deblurring, Monodepth demos

* Update bert_named_entity_recognition, bert_question_answering demos

* Move logging to OpenvinoAdapter, add class Core

* Ngraph removed from YOLO

* Fix

* Update human_pose_estimation_demo + Openpose wrapper

* Update hpe_associative_embedding wrapper, bert_question_answering_embedding demo

* Fix in object_detection_demo

* Delete extra EOF

* Update yolo.py

* Remove int2float(), add list of openvino precisions

* Small fixes

* Suggestions

* Add ability to read network from byte buffer.

* Update demos/common/python/openvino/model_zoo/model_api/adapters/openvino_adapter.py

* Update demos/common/python/openvino/model_zoo/model_api/adapters/openvino_adapter.py

* Remove trailing whitespaces

Co-authored-by: eizamaliev <[email protected]>

Author: anzhella-pankratova

demos/bert_named_entity_recognition_demo/python/bert_named_entity_recognition_demo.py

@@ -23,17 +23,15 @@
 from pathlib import Path
 from time import perf_counter
 
-from openvino.inference_engine import IECore, get_version
-
 sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python'))
 sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python/openvino/model_zoo'))
 
 from html_reader import get_paragraphs
-from helpers import log_runtime_settings
 
 from model_api.models import BertNamedEntityRecognition
 from model_api.models.tokens_bert import text_to_tokens, load_vocab_file
-from model_api.pipelines import get_user_config, parse_devices, AsyncPipeline
+from model_api.pipelines import get_user_config, AsyncPipeline
+from model_api.adapters import create_core, OpenvinoAdapter, RemoteAdapter
 
 log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout)
 
@@ -52,6 +50,8 @@ def build_argparser():
     args.add_argument("-i", "--input", help="Required. URL to a page with context",
                       action='append',
                       required=True, type=str)
+    args.add_argument('--adapter', help='Optional. Specify the model adapter. Default is openvino.',
+                      default='openvino', type=str, choices=('openvino', 'remote'))
     args.add_argument("--input_names",
                       help="Optional. Inputs names for the network. "
                            "Default values are \"input_ids,attention_mask,token_type_ids\" ",
@@ -105,21 +105,21 @@ def main():
     preprocessing_total_time = (perf_counter() - preprocessing_start_time) * 1e3
     source = tuple(zip(sentences, preprocessed_sentences))
 
-    log.info('OpenVINO Inference Engine')
-    log.info('\tbuild: {}'.format(get_version()))
-    ie = IECore()
-
-    plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
+    if args.adapter == 'openvino':
+        plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
+        model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
+                                        max_num_requests=args.num_infer_requests)
+    elif args.adapter == 'remote':
+        log.info('Reading model {}'.format(args.model))
+        serving_config = {"address": "localhost", "port": 9000}
+        model_adapter = RemoteAdapter(args.model, serving_config)
 
-    log.info('Reading model {}'.format(args.model))
-    model = BertNamedEntityRecognition(ie, args.model, vocab, args.input_names)
+    model = BertNamedEntityRecognition(model_adapter, vocab, args.input_names)
     if max_sentence_length > model.max_length:
         model.reshape(max_sentence_length)
+    model.log_layers_info()
 
-    pipeline = AsyncPipeline(ie, model, plugin_config,
-                             device=args.device, max_num_requests=args.num_infer_requests)
-    log.info('The model {} is loaded to {}'.format(args.model, args.device))
-    log_runtime_settings(pipeline.exec_net, set(parse_devices(args.device)))
+    pipeline = AsyncPipeline(model)
 
     next_sentence_id = 0
     next_sentence_id_to_show = 0

demos/bert_question_answering_demo/python/bert_question_answering_demo.py

@@ -23,17 +23,16 @@
 from time import perf_counter
 
 import numpy as np
-from openvino.inference_engine import IECore, get_version
 
 sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python'))
 sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python/openvino/model_zoo'))
 
 from html_reader import get_paragraphs
-from helpers import log_runtime_settings
 
 from model_api.models import BertQuestionAnswering
 from model_api.models.tokens_bert import text_to_tokens, load_vocab_file, ContextWindow
-from model_api.pipelines import get_user_config, parse_devices, AsyncPipeline
+from model_api.pipelines import get_user_config, AsyncPipeline
+from model_api.adapters import create_core, OpenvinoAdapter, RemoteAdapter
 
 log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout)
 
@@ -49,6 +48,8 @@ def build_argparser():
     args.add_argument("-i", "--input", help="Required. URL to a page with context",
                       action='append',
                       required=True, type=str)
+    args.add_argument('--adapter', help='Optional. Specify the model adapter. Default is openvino.',
+                      default='openvino', type=str, choices=('openvino', 'remote'))
     args.add_argument("--questions", type=str, nargs='+', metavar='QUESTION', help="Optional. Prepared questions")
     args.add_argument("--input_names",
                       help="Optional. Inputs names for the network. "
@@ -162,14 +163,16 @@ def main():
     c_tokens = text_to_tokens(context.lower(), vocab)
     total_latency = (perf_counter() - preprocessing_start_time) * 1e3
 
-    log.info('OpenVINO Inference Engine')
-    log.info('\tbuild: {}'.format(get_version()))
-    ie = IECore()
+    if args.adapter == 'openvino':
+        plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
+        model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config,
+                                        max_num_requests=args.num_infer_requests)
+    elif args.adapter == 'remote':
+        log.info('Reading model {}'.format(args.model))
+        serving_config = {"address": "localhost", "port": 9000}
+        model_adapter = RemoteAdapter(args.model, serving_config)
 
-    plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
-
-    log.info('Reading model {}'.format(args.model))
-    model = BertQuestionAnswering(ie, args.model, vocab, args.input_names, args.output_names,
+    model = BertQuestionAnswering(model_adapter, vocab, args.input_names, args.output_names,
                                   args.max_answer_token_num, args.model_squad_ver)
     if args.reshape:
         # find the closest multiple of 64, if it is smaller than current network's sequence length, do reshape
@@ -183,11 +186,9 @@ def main():
         else:
             log.debug("\tSkipping network reshaping,"
                       " as (context length + max question length) exceeds the current (input) network sequence length")
+    model.log_layers_info()
 
-    pipeline = AsyncPipeline(ie, model, plugin_config,
-                             device=args.device, max_num_requests=args.num_infer_requests)
-    log.info('The model {} is loaded to {}'.format(args.model, args.device))
-    log_runtime_settings(pipeline.exec_net, set(parse_devices(args.device)))
+    pipeline = AsyncPipeline(model)
 
     if args.questions:
         def questions():

demos/bert_question_answering_embedding_demo/python/bert_question_answering_embedding_demo.py

@@ -23,17 +23,16 @@
 from time import perf_counter
 
 import numpy as np
-from openvino.inference_engine import IECore, get_version
 
 sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python'))
 sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python/openvino/model_zoo'))
 
 from html_reader import get_paragraphs
-from helpers import log_runtime_settings
 
 from model_api.models import BertEmbedding, BertQuestionAnswering
 from model_api.models.tokens_bert import text_to_tokens, load_vocab_file, ContextWindow
-from model_api.pipelines import get_user_config, parse_devices, AsyncPipeline
+from model_api.pipelines import get_user_config, AsyncPipeline
+from model_api.adapters import create_core, OpenvinoAdapter
 
 log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout)
 
@@ -169,14 +168,12 @@ def main():
     visualizer = Visualizer(args.colors)
     total_latency = (perf_counter() - vocab_start_time) * 1e3
 
-    log.info('OpenVINO Inference Engine')
-    log.info('\tbuild: {}'.format(get_version()))
-    ie = IECore()
-
+    ie = create_core()
     plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
-
-    log.info('Reading Bert Embedding model {}'.format(args.model_emb))
-    model_emb = BertEmbedding(ie, args.model_emb, vocab, args.input_names_emb)
+    model_emb_adapter = OpenvinoAdapter(ie, args.model_emb, device=args.device, plugin_config=plugin_config,
+                                        max_num_requests=args.num_infer_requests)
+    model_emb = BertEmbedding(model_emb_adapter, vocab, args.input_names_emb)
+    model_emb.log_layers_info()
 
     # reshape BertEmbedding model to infer short questions and long contexts
     max_len_context = 384
@@ -185,21 +182,17 @@ def main():
     for new_length in [max_len_question, max_len_context]:
         model_emb.reshape(new_length)
         if new_length == max_len_question:
-            emb_exec_net = ie.load_network(model_emb.net, args.device)
+            emb_exec_net = ie.load_network(model_emb_adapter.net, args.device)
         else:
-            emb_pipeline = AsyncPipeline(
-                ie, model_emb, plugin_config, device=args.device, max_num_requests=args.num_infer_requests
-            )
-    log.info('The Bert Embedding model {} is loaded to {}'.format(args.model_emb, args.device))
-    log_runtime_settings(emb_pipeline.exec_net, set(parse_devices(args.device)))
+            emb_pipeline = AsyncPipeline(model_emb)
 
     if args.model_qa:
-        log.info('Reading Question Answering model {}'.format(args.model_qa))
-        model_qa = BertQuestionAnswering(ie, args.model_qa, vocab, args.input_names_qa, args.output_names_qa,
+        model_qa_adapter = OpenvinoAdapter(ie, args.model_qa, device=args.device, plugin_config=plugin_config,
+                                           max_num_requests=args.num_infer_requests)
+        model_qa = BertQuestionAnswering(model_qa_adapter, vocab, args.input_names_qa, args.output_names_qa,
                                          args.max_answer_token_num, args.model_qa_squad_ver)
-        qa_pipeline = AsyncPipeline(ie, model_qa, plugin_config, device=args.device, max_num_requests=args.num_infer_requests)
-        log_runtime_settings(qa_pipeline.exec_net, set(parse_devices(args.device)))
-        log.info('The Question Answering model {} is loaded to {}'.format(args.model_qa, args.device))
+        model_qa.log_layers_info()
+        qa_pipeline = AsyncPipeline(model_qa)
 
     log.info("\t\tStage 1    (Calc embeddings for the context)")
     contexts_all = []

demos/common/python/helpers.py

@@ -26,26 +26,6 @@ def resolution(value):
         raise RuntimeError('Сorrect format of --output_resolution parameter is "width"x"height".')
     return result
 
-def log_blobs_info(model):
-    for name, layer in model.net.input_info.items():
-        log.info('\tInput blob: {}, shape: {}, precision: {}'.format(name, layer.input_data.shape, layer.precision))
-    for name, layer in model.net.outputs.items():
-        log.info('\tOutput blob: {}, shape: {}, precision: {}'.format(name, layer.shape, layer.precision))
-
-def log_runtime_settings(exec_net, devices):
-    if 'AUTO' not in devices:
-        for device in devices:
-            try:
-                nstreams = exec_net.get_config(device + '_THROUGHPUT_STREAMS')
-                log.info('\tDevice: {}'.format(device))
-                log.info('\t\tNumber of streams: {}'.format(nstreams))
-                if device == 'CPU':
-                    nthreads = exec_net.get_config('CPU_THREADS_NUM')
-                    log.info('\t\tNumber of threads: {}'.format(nthreads if int(nthreads) else 'AUTO'))
-            except RuntimeError:
-                pass
-    log.info('\tNumber of network infer requests: {}'.format(len(exec_net.requests)))
-
 def log_latency_per_stage(*pipeline_metrics):
     stages = ('Decoding', 'Preprocessing', 'Inference', 'Postprocessing', 'Rendering')
     for stage, latency in zip(stages, pipeline_metrics):

demos/common/python/openvino/model_zoo/model_api/adapters/__init__.py

@@ -0,0 +1,25 @@
+"""
+ Copyright (C) 2021 Intel Corporation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+"""
+
+
+from .openvino_adapter import create_core, OpenvinoAdapter
+from .remote_adapter import RemoteAdapter
+
+__all__ = [
+    'create_core',
+    'OpenvinoAdapter',
+    'RemoteAdapter',
+]