diff --git a/src/cpp/include/adapters/inference_adapter.h b/src/cpp/include/adapters/inference_adapter.h index 9911f655..e38a7780 100644 --- a/src/cpp/include/adapters/inference_adapter.h +++ b/src/cpp/include/adapters/inference_adapter.h @@ -31,11 +31,11 @@ class InferenceAdapter { virtual void awaitAll() = 0; virtual void awaitAny() = 0; virtual size_t getNumAsyncExecutors() const = 0; - virtual void loadModel(const std::shared_ptr& model, - ov::Core& core, + virtual void loadModel(const std::string& modelPath, const std::string& device = "", - const ov::AnyMap& compilationConfig = {}, - size_t max_num_requests = 0) = 0; + const ov::AnyMap& adapterConfig = {}, + bool preCompile = true) = 0; + virtual void compileModel(const std::string& device = "", const ov::AnyMap& adapterConfig = {}) = 0; virtual ov::PartialShape getInputShape(const std::string& inputName) const = 0; virtual ov::PartialShape getOutputShape(const std::string& inputName) const = 0; virtual ov::element::Type_t getInputDatatype(const std::string& inputName) const = 0; diff --git a/src/cpp/include/adapters/openvino_adapter.h b/src/cpp/include/adapters/openvino_adapter.h index 7713ae93..31697cb1 100644 --- a/src/cpp/include/adapters/openvino_adapter.h +++ b/src/cpp/include/adapters/openvino_adapter.h @@ -25,11 +25,11 @@ class OpenVINOInferenceAdapter : public InferenceAdapter { virtual bool isReady(); virtual void awaitAll(); virtual void awaitAny(); - virtual void loadModel(const std::shared_ptr& model, - ov::Core& core, + virtual void loadModel(const std::string& modelPath, const std::string& device = "", - const ov::AnyMap& compilationConfig = {}, - size_t max_num_requests = 1) override; + const ov::AnyMap& adapterConfig = {}, + bool preCompile = true) override; + virtual void compileModel(const std::string& device = "", const ov::AnyMap& adapterConfig = {}) override; virtual size_t getNumAsyncExecutors() const; virtual ov::PartialShape getInputShape(const std::string& inputName) const override; virtual ov::PartialShape getOutputShape(const std::string& outputName) const override; @@ -39,6 +39,8 @@ class OpenVINOInferenceAdapter : public InferenceAdapter { virtual std::vector getOutputNames() const override; virtual const ov::AnyMap& getModelConfig() const override; + void applyModelTransform(std::function&)> t); + protected: void initInputsOutputs(); @@ -48,7 +50,6 @@ class OpenVINOInferenceAdapter : public InferenceAdapter { std::vector outputNames; std::unique_ptr asyncQueue; ov::AnyMap modelConfig; // the content of model_info section of rt_info - -public: + std::shared_ptr model; ov::CompiledModel compiledModel; }; diff --git a/src/cpp/src/adapters/openvino_adapter.cpp b/src/cpp/src/adapters/openvino_adapter.cpp index 920e5378..beabbb68 100644 --- a/src/cpp/src/adapters/openvino_adapter.cpp +++ b/src/cpp/src/adapters/openvino_adapter.cpp @@ -9,12 +9,16 @@ #include #include -void OpenVINOInferenceAdapter::loadModel(const std::shared_ptr& model, - ov::Core& core, - const std::string& device, - const ov::AnyMap& compilationConfig, - size_t max_num_requests) { - ov::AnyMap customCompilationConfig(compilationConfig); +#include "utils/config.h" + +void OpenVINOInferenceAdapter::compileModel(const std::string& device, const ov::AnyMap& adapterConfig) { + if (!model) { + throw std::runtime_error("Model is not loaded"); + } + size_t max_num_requests = 1; + max_num_requests = utils::get_from_any_maps("max_num_requests", adapterConfig, {}, max_num_requests); + + ov::AnyMap customCompilationConfig(adapterConfig); if (max_num_requests != 1) { if (customCompilationConfig.find("PERFORMANCE_HINT") == customCompilationConfig.end()) { customCompilationConfig["PERFORMANCE_HINT"] = ov::hint::PerformanceMode::THROUGHPUT; @@ -30,10 +34,31 @@ void OpenVINOInferenceAdapter::loadModel(const std::shared_ptr& } } + ov::Core core; compiledModel = core.compile_model(model, device, customCompilationConfig); asyncQueue = std::make_unique(compiledModel, max_num_requests); initInputsOutputs(); +} +void OpenVINOInferenceAdapter::loadModel(const std::string& modelPath, + const std::string& device, + const ov::AnyMap& adapterConfig, + bool preCompile) { + ov::Core core; + model = core.read_model(modelPath); + if (model->has_rt_info({"model_info"})) { + modelConfig = model->get_rt_info("model_info"); + } + if (preCompile) { + compileModel(device, adapterConfig); + } +} + +void OpenVINOInferenceAdapter::applyModelTransform(std::function&)> t) { + if (!model) { + throw std::runtime_error("Model is not loaded"); + } + t(model); if (model->has_rt_info({"model_info"})) { modelConfig = model->get_rt_info("model_info"); } diff --git a/src/cpp/src/tasks/anomaly.cpp b/src/cpp/src/tasks/anomaly.cpp index 4a0aa993..29f42d46 100644 --- a/src/cpp/src/tasks/anomaly.cpp +++ b/src/cpp/src/tasks/anomaly.cpp @@ -5,6 +5,11 @@ #include "utils/tensor.h" void Anomaly::serialize(std::shared_ptr& ov_model) { + if (utils::model_has_embedded_processing(ov_model)) { + std::cout << "model already was serialized" << std::endl; + return; + } + auto input = ov_model->inputs().front(); auto layout = ov::layout::get_layout(input); @@ -47,23 +52,21 @@ void Anomaly::serialize(std::shared_ptr& ov_model) { } Anomaly Anomaly::load(const std::string& model_path) { - auto core = ov::Core(); - std::shared_ptr model = core.read_model(model_path); + auto adapter = std::make_shared(); + adapter->loadModel(model_path, "", {}, false); + + std::string model_type; + model_type = utils::get_from_any_maps("model_type", adapter->getModelConfig(), {}, model_type); - if (model->has_rt_info("model_info", "model_type")) { - std::cout << "has model type in info: " << model->get_rt_info("model_info", "model_type") - << std::endl; + if (!model_type.empty()) { + std::cout << "has model type in info: " << model_type << std::endl; } else { throw std::runtime_error("Incorrect or unsupported model_type"); } - if (utils::model_has_embedded_processing(model)) { - std::cout << "model already was serialized" << std::endl; - } else { - serialize(model); - } - auto adapter = std::make_shared(); - adapter->loadModel(model, core, "AUTO"); + adapter->applyModelTransform(Anomaly::serialize); + adapter->compileModel("AUTO", {}); + return Anomaly(adapter); } diff --git a/src/cpp/src/tasks/classification.cpp b/src/cpp/src/tasks/classification.cpp index 0f93912f..3a74d341 100644 --- a/src/cpp/src/tasks/classification.cpp +++ b/src/cpp/src/tasks/classification.cpp @@ -83,6 +83,10 @@ std::vector get_non_xai_output_indices(const std::vector& ov_model) { + if (utils::model_has_embedded_processing(ov_model)) { + std::cout << "model already was serialized" << std::endl; + return; + } // --------------------------- Configure input & output ------------------------------------------------- // --------------------------- Prepare input ------------------------------------------------------ auto config = ov_model->has_rt_info("model_info") ? ov_model->get_rt_info("model_info") : ov::AnyMap{}; @@ -177,23 +181,18 @@ void Classification::serialize(std::shared_ptr& ov_model) { } Classification Classification::load(const std::string& model_path) { - auto core = ov::Core(); - std::shared_ptr model = core.read_model(model_path); + auto adapter = std::make_shared(); + adapter->loadModel(model_path, "", {}, false); - if (model->has_rt_info("model_info", "model_type")) { - std::cout << "has model type in info: " << model->get_rt_info("model_info", "model_type") - << std::endl; - } else { - throw std::runtime_error("Incorrect or unsupported model_type"); - } + std::string model_type; + model_type = utils::get_from_any_maps("model_type", adapter->getModelConfig(), {}, model_type); - if (utils::model_has_embedded_processing(model)) { - std::cout << "model already was serialized" << std::endl; - } else { - Classification::serialize(model); + if (model_type.empty() || model_type != "Classification") { + throw std::runtime_error("Incorrect or unsupported model_type, expected: Classification"); } - auto adapter = std::make_shared(); - adapter->loadModel(model, core, "AUTO"); + adapter->applyModelTransform(Classification::serialize); + adapter->compileModel("AUTO", {}); + return Classification(adapter); } diff --git a/src/cpp/src/tasks/detection.cpp b/src/cpp/src/tasks/detection.cpp index 3a4432a9..d05433fe 100644 --- a/src/cpp/src/tasks/detection.cpp +++ b/src/cpp/src/tasks/detection.cpp @@ -12,23 +12,19 @@ #include "utils/tensor.h" DetectionModel DetectionModel::load(const std::string& model_path, const ov::AnyMap& configuration) { - auto core = ov::Core(); - std::shared_ptr model = core.read_model(model_path); + auto adapter = std::make_shared(); + adapter->loadModel(model_path, "", {}, false); - if (model->has_rt_info("model_info", "model_type")) { - std::cout << "has model type in info: " << model->get_rt_info("model_info", "model_type") - << std::endl; - } else { - throw std::runtime_error("Incorrect or unsupported model_type"); - } + std::string model_type; + model_type = utils::get_from_any_maps("model_type", adapter->getModelConfig(), {}, model_type); + transform(model_type.begin(), model_type.end(), model_type.begin(), ::tolower); - if (utils::model_has_embedded_processing(model)) { - std::cout << "model already was serialized" << std::endl; - } else { - SSD::serialize(model); + if (model_type.empty() || model_type != "ssd") { + throw std::runtime_error("Incorrect or unsupported model_type, expected: ssd"); } - auto adapter = std::make_shared(); - adapter->loadModel(model, core, "AUTO"); + adapter->applyModelTransform(SSD::serialize); + adapter->compileModel("AUTO", {}); + return DetectionModel(std::make_unique(adapter), configuration); } diff --git a/src/cpp/src/tasks/detection/ssd.cpp b/src/cpp/src/tasks/detection/ssd.cpp index 2d656cad..3fb551dc 100644 --- a/src/cpp/src/tasks/detection/ssd.cpp +++ b/src/cpp/src/tasks/detection/ssd.cpp @@ -68,6 +68,10 @@ std::map SSD::preprocess(cv::Mat image) { } void SSD::serialize(std::shared_ptr ov_model) { + if (utils::model_has_embedded_processing(ov_model)) { + std::cout << "model already was serialized" << std::endl; + return; + } auto output_mode = ov_model->outputs().size() > 1 ? SSDOutputMode::multi : SSDOutputMode::single; auto input_tensor = ov_model->inputs()[0]; diff --git a/src/cpp/src/tasks/instance_segmentation.cpp b/src/cpp/src/tasks/instance_segmentation.cpp index c58290c2..a98cbc32 100644 --- a/src/cpp/src/tasks/instance_segmentation.cpp +++ b/src/cpp/src/tasks/instance_segmentation.cpp @@ -122,6 +122,10 @@ cv::Mat segm_postprocess(const SegmentedObject& box, const cv::Mat& unpadded, in } void InstanceSegmentation::serialize(std::shared_ptr& ov_model) { + if (utils::model_has_embedded_processing(ov_model)) { + std::cout << "model already was serialized" << std::endl; + return; + } if (ov_model->inputs().size() != 1) { throw std::logic_error("MaskRCNNModel model wrapper supports topologies with only 1 input"); } @@ -187,23 +191,18 @@ void InstanceSegmentation::serialize(std::shared_ptr& ov_model) { } InstanceSegmentation InstanceSegmentation::load(const std::string& model_path) { - auto core = ov::Core(); - std::shared_ptr model = core.read_model(model_path); - - if (model->has_rt_info("model_info", "model_type")) { - std::cout << "has model type in info: " << model->get_rt_info("model_info", "model_type") - << std::endl; - } else { - throw std::runtime_error("Incorrect or unsupported model_type"); - } + auto adapter = std::make_shared(); + adapter->loadModel(model_path, "", {}, false); - if (utils::model_has_embedded_processing(model)) { - std::cout << "model already was serialized" << std::endl; - } else { - serialize(model); + std::string model_type; + model_type = utils::get_from_any_maps("model_type", adapter->getModelConfig(), {}, model_type); + + if (model_type.empty() || model_type != "MaskRCNN") { + throw std::runtime_error("Incorrect or unsupported model_type, expected: MaskRCNN"); } - auto adapter = std::make_shared(); - adapter->loadModel(model, core, "AUTO"); + adapter->applyModelTransform(InstanceSegmentation::serialize); + adapter->compileModel("AUTO", {}); + return InstanceSegmentation(adapter); } diff --git a/src/cpp/src/tasks/semantic_segmentation.cpp b/src/cpp/src/tasks/semantic_segmentation.cpp index 39de5309..588045ad 100644 --- a/src/cpp/src/tasks/semantic_segmentation.cpp +++ b/src/cpp/src/tasks/semantic_segmentation.cpp @@ -21,27 +21,26 @@ cv::Mat get_activation_map(const cv::Mat& features) { } SemanticSegmentation SemanticSegmentation::load(const std::string& model_path) { - auto core = ov::Core(); - std::shared_ptr model = core.read_model(model_path); + auto adapter = std::make_shared(); + adapter->loadModel(model_path, "", {}, false); - if (model->has_rt_info("model_info", "model_type")) { - std::cout << "has model type in info: " << model->get_rt_info("model_info", "model_type") - << std::endl; - } else { - throw std::runtime_error("Incorrect or unsupported model_type"); - } + std::string model_type; + model_type = utils::get_from_any_maps("model_type", adapter->getModelConfig(), {}, model_type); - if (utils::model_has_embedded_processing(model)) { - std::cout << "model already was serialized" << std::endl; - } else { - SemanticSegmentation::serialize(model); + if (model_type.empty() || model_type != "Segmentation") { + throw std::runtime_error("Incorrect or unsupported model_type, expected: Segmentation"); } - auto adapter = std::make_shared(); - adapter->loadModel(model, core, "AUTO"); + adapter->applyModelTransform(SemanticSegmentation::serialize); + adapter->compileModel("AUTO", {}); + return SemanticSegmentation(adapter); } void SemanticSegmentation::serialize(std::shared_ptr& ov_model) { + if (utils::model_has_embedded_processing(ov_model)) { + std::cout << "model already was serialized" << std::endl; + return; + } if (ov_model->inputs().size() != 1) { throw std::logic_error("Segmentation model wrapper supports topologies with only 1 input"); }