Adjust cpp inference adapters for OVMS (#212)

* Adjust adapters for OVMS

This is required to allow no-copy inference with Model API within
Mediapipe graphs.

* Fix test build

* Fixes

Author: atobiszei

model_api/cpp/adapters/include/adapters/inference_adapter.h

@@ -24,6 +24,7 @@ class InferenceAdapter {
     virtual ~InferenceAdapter() = default;
 
     virtual InferenceOutput infer(const InferenceInput& input) = 0;
+    virtual void infer(const InferenceInput& input, InferenceOutput& output) = 0;
     virtual void setCallback(std::function<void(ov::InferRequest, CallbackData)> callback) = 0;
     virtual void inferAsync(const InferenceInput& input, CallbackData callback_args) = 0;
     virtual bool isReady() = 0;
@@ -36,6 +37,9 @@ class InferenceAdapter {
                            const ov::AnyMap& compilationConfig = {},
                            size_t max_num_requests = 0) = 0;
     virtual ov::PartialShape getInputShape(const std::string& inputName) const = 0;
+    virtual ov::PartialShape getOutputShape(const std::string& inputName) const = 0;
+    virtual ov::element::Type_t getInputDatatype(const std::string& inputName) const = 0;
+    virtual ov::element::Type_t getOutputDatatype(const std::string& outputName) const = 0;
     virtual std::vector<std::string> getInputNames() const = 0;
     virtual std::vector<std::string> getOutputNames() const = 0;
     virtual const ov::AnyMap& getModelConfig() const = 0;

model_api/cpp/adapters/include/adapters/openvino_adapter.h

@@ -19,6 +19,7 @@ class OpenVINOInferenceAdapter : public InferenceAdapter {
     OpenVINOInferenceAdapter() = default;
 
     virtual InferenceOutput infer(const InferenceInput& input) override;
+    virtual void infer(const InferenceInput& input, InferenceOutput& output) override;
     virtual void inferAsync(const InferenceInput& input, const CallbackData callback_args) override;
     virtual void setCallback(std::function<void(ov::InferRequest, const CallbackData)> callback);
     virtual bool isReady();
@@ -31,6 +32,9 @@ class OpenVINOInferenceAdapter : public InferenceAdapter {
                            size_t max_num_requests = 1) override;
     virtual size_t getNumAsyncExecutors() const;
     virtual ov::PartialShape getInputShape(const std::string& inputName) const override;
+    virtual ov::PartialShape getOutputShape(const std::string& outputName) const override;
+    virtual ov::element::Type_t getInputDatatype(const std::string& inputName) const override;
+    virtual ov::element::Type_t getOutputDatatype(const std::string& outputName) const override;
     virtual std::vector<std::string> getInputNames() const override;
     virtual std::vector<std::string> getOutputNames() const override;
     virtual const ov::AnyMap& getModelConfig() const override;

model_api/cpp/adapters/src/openvino_adapter.cpp

@@ -41,6 +41,20 @@ void OpenVINOInferenceAdapter::loadModel(const std::shared_ptr<const ov::Model>&
     }
 }
 
+void OpenVINOInferenceAdapter::infer(const InferenceInput& input, InferenceOutput& output) {
+    auto request = asyncQueue->operator[](asyncQueue->get_idle_request_id());
+    for (const auto& [name, tensor] : input) {
+        request.set_tensor(name, tensor);
+    }
+    for (const auto& [name, tensor] : output) {
+        request.set_tensor(name, tensor);
+    }
+    request.infer();
+    for (const auto& name : outputNames) {
+        output[name] = request.get_tensor(name);
+    }
+}
+
 InferenceOutput OpenVINOInferenceAdapter::infer(const InferenceInput& input) {
     auto request = asyncQueue->operator[](asyncQueue->get_idle_request_id());
     // Fill input blobs
@@ -87,6 +101,9 @@ size_t OpenVINOInferenceAdapter::getNumAsyncExecutors() const {
 ov::PartialShape OpenVINOInferenceAdapter::getInputShape(const std::string& inputName) const {
     return compiledModel.input(inputName).get_partial_shape();
 }
+ov::PartialShape OpenVINOInferenceAdapter::getOutputShape(const std::string& outputName) const {
+    return compiledModel.output(outputName).get_partial_shape();
+}
 
 void OpenVINOInferenceAdapter::initInputsOutputs() {
     for (const auto& input : compiledModel.inputs()) {
@@ -97,6 +114,12 @@ void OpenVINOInferenceAdapter::initInputsOutputs() {
         outputNames.push_back(output.get_any_name());
     }
 }
+ov::element::Type_t OpenVINOInferenceAdapter::getInputDatatype(const std::string&) const {
+    throw std::runtime_error("Not implemented");
+}
+ov::element::Type_t OpenVINOInferenceAdapter::getOutputDatatype(const std::string&) const {
+    throw std::runtime_error("Not implemented");
+}
 
 std::vector<std::string> OpenVINOInferenceAdapter::getInputNames() const {
     return inputNames;