diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..65ee5d21 --- /dev/null +++ b/.clang-format @@ -0,0 +1,28 @@ +BasedOnStyle: Google +IndentWidth: 4 +UseTab: Never +ColumnLimit: 120 + +Language: Cpp +Standard: Cpp11 + +AccessModifierOffset: -4 +AlignConsecutiveMacros: true +AllowAllArgumentsOnNextLine: false +AllowAllConstructorInitializersOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: Empty +AllowShortLoopsOnASingleLine: false +AlwaysBreakBeforeMultilineStrings: false +BinPackArguments: false +BinPackParameters: false +CommentPragmas: "^#" +DerivePointerAlignment: false +FixNamespaceComments: true +IndentCaseLabels: false +IndentPPDirectives: AfterHash +ForEachMacros: + - foreach + - FOREACH_CHILD diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml index 19136d08..d4299024 100644 --- a/.github/workflows/pre_commit.yml +++ b/.github/workflows/pre_commit.yml @@ -23,6 +23,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.10" + - name: Install clang-format + run: sudo apt-get install -y clang-format-10 - name: Install dependencies run: pip install 'model_api/python/.[full]' - name: Run pre-commit checks diff --git a/.github/workflows/test_precommit.yml b/.github/workflows/test_precommit.yml index 53bc9b84..208e722a 100644 --- a/.github/workflows/test_precommit.yml +++ b/.github/workflows/test_precommit.yml @@ -9,11 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: git --no-pager diff --check $(git hash-object -t tree /dev/null) - run: git --no-pager diff --check $(git hash-object -t tree /dev/null) - - name: Prohibit non ASCII chars in file names - run: test $(git diff --name-only --diff-filter=A -z $(git hash-object -t tree /dev/null) | LC_ALL=C tr -d '[ -~]\0' | wc -c) == 0 - - run: "! git grep -n '[^ -~]' -- ':(exclude)model_api/python/README.md'" - uses: actions/setup-python@v4 with: python-version: 3.9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d61adae..9247919d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,10 @@ repos: rev: v0.41.0 hooks: - id: markdownlint + + # C++ code quality + - repo: https://github.com/cpp-linter/cpp-linter-hooks + rev: v0.5.1 + hooks: + - id: clang-format + args: [--style=file, --version=10] diff --git a/examples/cpp/asynchronous_api/main.cpp b/examples/cpp/asynchronous_api/main.cpp index 4ff85200..d5ad738b 100644 --- a/examples/cpp/asynchronous_api/main.cpp +++ b/examples/cpp/asynchronous_api/main.cpp @@ -13,24 +13,20 @@ // See the License for the specific language governing permissions and // limitations under the License. */ - +#include +#include +#include #include #include #include #include #include -#include -#include - #include #include #include - -#include -#include -#include - +#include +#include int main(int argc, char* argv[]) try { if (argc != 3) { @@ -43,43 +39,47 @@ int main(int argc, char* argv[]) try { } // Instantiate Object Detection model - auto model = DetectionModel::create_model(argv[1], {}, "", false); // works with SSD models. Download it using Python Model API - //Define number of parallel infer requests. Is this number is set to 0, OpenVINO will determine it automatically to obtain optimal performance. + auto model = DetectionModel::create_model(argv[1], + {}, + "", + false); // works with SSD models. Download it using Python Model API + // Define number of parallel infer requests. Is this number is set to 0, OpenVINO will determine it automatically to + // obtain optimal performance. size_t num_requests = 0; static ov::Core core; model->load(core, "CPU", num_requests); std::cout << "Async inference will be carried out by " << model->getNumAsyncExecutors() << " parallel executors\n"; - //Prepare batch data + // Prepare batch data std::vector data; for (size_t i = 0; i < 3; i++) { data.push_back(ImageInputData(image)); } - //Batch inference is done by processing batch with num_requests parallel infer requests + // Batch inference is done by processing batch with num_requests parallel infer requests std::cout << "Starting batch inference\n"; auto results = model->inferBatch(data); std::cout << "Batch mode inference results:\n"; for (const auto& result : results) { for (auto& obj : result->objects) { - std::cout << " " << std::left << std::setw(9) << obj.confidence << " " << obj.label << "\n"; + std::cout << " " << std::left << std::setw(9) << obj.confidence << " " << obj.label << "\n"; } std::cout << std::string(10, '-') << "\n"; } std::cout << "Batch mode inference done\n"; std::cout << "Async mode inference results:\n"; - //Set callback to grab results once the inference is done - model->setCallback([](std::unique_ptr result, const ov::AnyMap& callback_args){ + // Set callback to grab results once the inference is done + model->setCallback([](std::unique_ptr result, const ov::AnyMap& callback_args) { auto det_result = std::unique_ptr(static_cast(result.release())); - //callback_args can contain arbitrary data + // callback_args can contain arbitrary data size_t id = callback_args.find("id")->second.as(); std::cout << "Request with id " << id << " is finished\n"; for (auto& obj : det_result->objects) { - std::cout << " " << std::left << std::setw(9) << obj.confidence << " " << obj.label << "\n"; + std::cout << " " << std::left << std::setw(9) << obj.confidence << " " << obj.label << "\n"; } std::cout << std::string(10, '-') << "\n"; }); @@ -88,7 +88,6 @@ int main(int argc, char* argv[]) try { model->inferAsync(image, {{"id", i}}); } model->awaitAll(); - } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; diff --git a/examples/cpp/synchronous_api/main.cpp b/examples/cpp/synchronous_api/main.cpp index f889fa72..81997109 100644 --- a/examples/cpp/synchronous_api/main.cpp +++ b/examples/cpp/synchronous_api/main.cpp @@ -14,23 +14,21 @@ // limitations under the License. */ +#include +#include +#include #include #include #include #include #include -#include -#include - #include #include #include #include - -#include -#include -#include +#include +#include int main(int argc, char* argv[]) try { if (argc != 3) { @@ -43,16 +41,16 @@ int main(int argc, char* argv[]) try { } // Instantiate Object Detection model - auto model = DetectionModel::create_model(argv[1]); // works with SSD models. Download it using Python Model API + auto model = DetectionModel::create_model(argv[1]); // works with SSD models. Download it using Python Model API // Run the inference auto result = model->infer(image); // Process detections for (auto& obj : result->objects) { - std::cout << " " << std::left << std::setw(9) << obj.label << " | " << std::setw(10) << obj.confidence - << " | " << std::setw(4) << int(obj.x) << " | " << std::setw(4) << int(obj.y) << " | " - << std::setw(4) << int(obj.x + obj.width) << " | " << std::setw(4) << int(obj.y + obj.height) << "\n"; + std::cout << " " << std::left << std::setw(9) << obj.label << " | " << std::setw(10) << obj.confidence << " | " + << std::setw(4) << int(obj.x) << " | " << std::setw(4) << int(obj.y) << " | " << std::setw(4) + << int(obj.x + obj.width) << " | " << std::setw(4) << int(obj.y + obj.height) << "\n"; } } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/model_api/cpp/adapters/include/adapters/inference_adapter.h b/model_api/cpp/adapters/include/adapters/inference_adapter.h index c4352b75..cec2ffd8 100644 --- a/model_api/cpp/adapters/include/adapters/inference_adapter.h +++ b/model_api/cpp/adapters/include/adapters/inference_adapter.h @@ -15,13 +15,12 @@ */ #pragma once -#include #include -#include #include #include - #include +#include +#include struct InputData; struct InferenceResult; @@ -31,9 +30,7 @@ using InferenceInput = std::map; using CallbackData = std::shared_ptr; // The interface doesn't have implementation -class InferenceAdapter -{ - +class InferenceAdapter { public: virtual ~InferenceAdapter() = default; @@ -44,8 +41,10 @@ class InferenceAdapter virtual void awaitAll() = 0; virtual void awaitAny() = 0; virtual size_t getNumAsyncExecutors() const = 0; - virtual void loadModel(const std::shared_ptr& model, ov::Core& core, - const std::string& device = "", const ov::AnyMap& compilationConfig = {}, + virtual void loadModel(const std::shared_ptr& model, + ov::Core& core, + const std::string& device = "", + const ov::AnyMap& compilationConfig = {}, size_t max_num_requests = 0) = 0; virtual ov::PartialShape getInputShape(const std::string& inputName) const = 0; virtual std::vector getInputNames() const = 0; diff --git a/model_api/cpp/adapters/include/adapters/openvino_adapter.h b/model_api/cpp/adapters/include/adapters/openvino_adapter.h index c4df9b99..1ac53973 100644 --- a/model_api/cpp/adapters/include/adapters/openvino_adapter.h +++ b/model_api/cpp/adapters/include/adapters/openvino_adapter.h @@ -15,19 +15,17 @@ */ #pragma once -#include #include -#include #include -#include #include +#include +#include +#include #include "adapters/inference_adapter.h" #include "utils/async_infer_queue.hpp" -class OpenVINOInferenceAdapter :public InferenceAdapter -{ - +class OpenVINOInferenceAdapter : public InferenceAdapter { public: OpenVINOInferenceAdapter() = default; @@ -37,9 +35,11 @@ class OpenVINOInferenceAdapter :public InferenceAdapter virtual bool isReady(); virtual void awaitAll(); virtual void awaitAny(); - virtual void loadModel(const std::shared_ptr& model, ov::Core& core, - const std::string& device = "", const ov::AnyMap& compilationConfig = {}, - size_t max_num_requests = 1) override; + virtual void loadModel(const std::shared_ptr& model, + ov::Core& core, + const std::string& device = "", + const ov::AnyMap& compilationConfig = {}, + size_t max_num_requests = 1) override; virtual size_t getNumAsyncExecutors() const; virtual ov::PartialShape getInputShape(const std::string& inputName) const override; virtual std::vector getInputNames() const override; @@ -50,10 +50,10 @@ class OpenVINOInferenceAdapter :public InferenceAdapter void initInputsOutputs(); protected: - //Depends on the implementation details but we should share the model state in this class + // Depends on the implementation details but we should share the model state in this class std::vector inputNames; std::vector outputNames; ov::CompiledModel compiledModel; std::unique_ptr asyncQueue; - ov::AnyMap modelConfig; // the content of model_info section of rt_info + ov::AnyMap modelConfig; // the content of model_info section of rt_info }; diff --git a/model_api/cpp/adapters/src/openvino_adapter.cpp b/model_api/cpp/adapters/src/openvino_adapter.cpp index 39632e3d..3ead7918 100644 --- a/model_api/cpp/adapters/src/openvino_adapter.cpp +++ b/model_api/cpp/adapters/src/openvino_adapter.cpp @@ -15,13 +15,17 @@ */ #include "adapters/openvino_adapter.h" + #include -#include #include +#include #include -void OpenVINOInferenceAdapter::loadModel(const std::shared_ptr& model, ov::Core& core, - const std::string& device, const ov::AnyMap& compilationConfig, size_t max_num_requests) { +void OpenVINOInferenceAdapter::loadModel(const std::shared_ptr& model, + ov::Core& core, + const std::string& device, + const ov::AnyMap& compilationConfig, + size_t max_num_requests) { slog::info << "Loading model to the plugin" << slog::endl; ov::AnyMap customCompilationConfig(compilationConfig); if (max_num_requests != 1) { @@ -33,8 +37,7 @@ void OpenVINOInferenceAdapter::loadModel(const std::shared_ptr& customCompilationConfig["PERFORMANCE_HINT_NUM_REQUESTS"] = ov::hint::num_requests(max_num_requests); } } - } - else { + } else { if (customCompilationConfig.find("PERFORMANCE_HINT") == customCompilationConfig.end()) { customCompilationConfig["PERFORMANCE_HINT"] = ov::hint::PerformanceMode::LATENCY; } diff --git a/model_api/cpp/models/include/models/classification_model.h b/model_api/cpp/models/include/models/classification_model.h index f14318c6..484123e5 100644 --- a/model_api/cpp/models/include/models/classification_model.h +++ b/model_api/cpp/models/include/models/classification_model.h @@ -17,11 +17,11 @@ #pragma once #include +#include #include #include -#include -#include #include +#include #include "models/image_model.h" @@ -37,7 +37,7 @@ struct HierarchicalConfig { std::map label_to_idx; std::vector> label_tree_edges; std::vector> all_groups; - std::map> head_idx_to_logits_range; + std::map> head_idx_to_logits_range; std::map logit_idx_to_label; size_t num_multiclass_heads; size_t num_multilabel_heads; @@ -48,53 +48,55 @@ struct HierarchicalConfig { }; class SimpleLabelsGraph { - public: - SimpleLabelsGraph() = default; - SimpleLabelsGraph(const std::vector& vertices_); - void add_edge(const std::string& parent, const std::string& child); - std::vector get_children(const std::string& label) const; - std::string get_parent(const std::string& label) const; - std::vector get_ancestors(const std::string& label) const; - std::vector get_labels_in_topological_order(); - - protected: - std::vector vertices; - std::unordered_map> adj; - std::unordered_map parents_map; - bool t_sort_cache_valid = false; - std::vector topological_order_cache; - - std::vector topological_sort(); +public: + SimpleLabelsGraph() = default; + SimpleLabelsGraph(const std::vector& vertices_); + void add_edge(const std::string& parent, const std::string& child); + std::vector get_children(const std::string& label) const; + std::string get_parent(const std::string& label) const; + std::vector get_ancestors(const std::string& label) const; + std::vector get_labels_in_topological_order(); + +protected: + std::vector vertices; + std::unordered_map> adj; + std::unordered_map parents_map; + bool t_sort_cache_valid = false; + std::vector topological_order_cache; + + std::vector topological_sort(); }; class GreedyLabelsResolver { - public: - GreedyLabelsResolver() = default; - GreedyLabelsResolver(const HierarchicalConfig&); - - virtual std::map resolve_labels(const std::vector>& labels, - const std::vector& scores); - protected: - std::map label_to_idx; - std::vector> label_relations; - std::vector> label_groups; - - std::string get_parent(const std::string& label); - std::vector get_predecessors(const std::string& label, const std::vector& candidates); +public: + GreedyLabelsResolver() = default; + GreedyLabelsResolver(const HierarchicalConfig&); + + virtual std::map resolve_labels(const std::vector>& labels, + const std::vector& scores); + +protected: + std::map label_to_idx; + std::vector> label_relations; + std::vector> label_groups; + + std::string get_parent(const std::string& label); + std::vector get_predecessors(const std::string& label, const std::vector& candidates); }; class ProbabilisticLabelsResolver : public GreedyLabelsResolver { - public: - ProbabilisticLabelsResolver() = default; - ProbabilisticLabelsResolver(const HierarchicalConfig&); - - virtual std::map resolve_labels(const std::vector>& labels, - const std::vector& scores); - std::unordered_map add_missing_ancestors(const std::unordered_map&) const; - std::map resolve_exclusive_labels(const std::unordered_map&) const; - void suppress_descendant_output(std::map&); - protected: - SimpleLabelsGraph label_tree; +public: + ProbabilisticLabelsResolver() = default; + ProbabilisticLabelsResolver(const HierarchicalConfig&); + + virtual std::map resolve_labels(const std::vector>& labels, + const std::vector& scores); + std::unordered_map add_missing_ancestors(const std::unordered_map&) const; + std::map resolve_exclusive_labels(const std::unordered_map&) const; + void suppress_descendant_output(std::map&); + +protected: + SimpleLabelsGraph label_tree; }; class ClassificationModel : public ImageModel { @@ -102,7 +104,10 @@ class ClassificationModel : public ImageModel { ClassificationModel(std::shared_ptr& model, const ov::AnyMap& configuration); ClassificationModel(std::shared_ptr& adapter, const ov::AnyMap& configuration = {}); - static std::unique_ptr create_model(const std::string& modelFile, const ov::AnyMap& configuration = {}, bool preload = true, const std::string& device = "AUTO"); + static std::unique_ptr create_model(const std::string& modelFile, + const ov::AnyMap& configuration = {}, + bool preload = true, + const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); std::unique_ptr postprocess(InferenceResult& infResult) override; diff --git a/model_api/cpp/models/include/models/detection_model.h b/model_api/cpp/models/include/models/detection_model.h index 49aa504b..eff58797 100644 --- a/model_api/cpp/models/include/models/detection_model.h +++ b/model_api/cpp/models/include/models/detection_model.h @@ -17,6 +17,7 @@ #pragma once #include + #include "models/image_model.h" struct DetectionResult; diff --git a/model_api/cpp/models/include/models/detection_model_yolo.h b/model_api/cpp/models/include/models/detection_model_yolo.h index fb291203..e365339a 100644 --- a/model_api/cpp/models/include/models/detection_model_yolo.h +++ b/model_api/cpp/models/include/models/detection_model_yolo.h @@ -20,11 +20,10 @@ #include #include -#include -#include - #include #include +#include +#include #include "models/detection_model_ext.h" @@ -53,7 +52,7 @@ class ModelYolo : public DetectionModelExt { }; public: - enum class YoloVersion : size_t { YOLO_V1V2=0, YOLO_V3, YOLO_V4, YOLO_V4_TINY, YOLOF }; + enum class YoloVersion : size_t { YOLO_V1V2 = 0, YOLO_V3, YOLO_V4, YOLO_V4_TINY, YOLOF }; ModelYolo(std::shared_ptr& model, const ov::AnyMap& configuration); ModelYolo(std::shared_ptr& adapter); @@ -90,6 +89,7 @@ class YOLOv5 : public DetectionModelExt { void updateModelInfo() override; void init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority); bool agnostic_nms = false; + public: YOLOv5(std::shared_ptr& model, const ov::AnyMap& configuration); YOLOv5(std::shared_ptr& adapter); diff --git a/model_api/cpp/models/include/models/detection_model_yolov3_onnx.h b/model_api/cpp/models/include/models/detection_model_yolov3_onnx.h index 504db78b..e650d2d6 100644 --- a/model_api/cpp/models/include/models/detection_model_yolov3_onnx.h +++ b/model_api/cpp/models/include/models/detection_model_yolov3_onnx.h @@ -16,14 +16,13 @@ #pragma once +#include #include #include -#include - #include "models/detection_model.h" -class ModelYoloV3ONNX: public DetectionModel { +class ModelYoloV3ONNX : public DetectionModel { public: ModelYoloV3ONNX(std::shared_ptr& model, const ov::AnyMap& configuration); ModelYoloV3ONNX(std::shared_ptr& adapter); diff --git a/model_api/cpp/models/include/models/detection_model_yolox.h b/model_api/cpp/models/include/models/detection_model_yolox.h index e6421a80..5e22b1f0 100644 --- a/model_api/cpp/models/include/models/detection_model_yolox.h +++ b/model_api/cpp/models/include/models/detection_model_yolox.h @@ -16,15 +16,13 @@ #pragma once #include +#include #include #include -#include - #include "models/detection_model_ext.h" - -class ModelYoloX: public DetectionModelExt { +class ModelYoloX : public DetectionModelExt { public: ModelYoloX(std::shared_ptr& model, const ov::AnyMap& configuration); ModelYoloX(std::shared_ptr& adapter); diff --git a/model_api/cpp/models/include/models/image_model.h b/model_api/cpp/models/include/models/image_model.h index a154d9c6..5d1af76c 100644 --- a/model_api/cpp/models/include/models/image_model.h +++ b/model_api/cpp/models/include/models/image_model.h @@ -20,8 +20,8 @@ #include #include -#include "models/model_base.h" #include "models/input_data.h" +#include "models/model_base.h" #include "utils/image_utils.h" namespace ov { @@ -49,16 +49,16 @@ class ImageModel : public ModelBase { std::shared_ptr preprocess(const InputData& inputData, InferenceInput& input) override; static std::vector loadLabels(const std::string& labelFilename); std::shared_ptr embedProcessing(std::shared_ptr& model, - const std::string& inputName, - const ov::Layout&, - RESIZE_MODE resize_mode, - const cv::InterpolationFlags interpolationMode, - const ov::Shape& targetShape, - uint8_t pad_value, - bool brg2rgb, - const std::vector& mean, - const std::vector& scale, - const std::type_info& dtype = typeid(int)); + const std::string& inputName, + const ov::Layout&, + RESIZE_MODE resize_mode, + const cv::InterpolationFlags interpolationMode, + const ov::Shape& targetShape, + uint8_t pad_value, + bool brg2rgb, + const std::vector& mean, + const std::vector& scale, + const std::type_info& dtype = typeid(int)); virtual void inferAsync(const ImageInputData& inputData, const ov::AnyMap& callback_args = {}); std::unique_ptr inferImage(const ImageInputData& inputData); std::vector> inferBatchImage(const std::vector& inputData); @@ -74,7 +74,7 @@ class ImageModel : public ModelBase { std::vector labels = {}; bool useAutoResize = false; - bool embedded_processing = false; // flag in model_info that pre/postprocessing embedded + bool embedded_processing = false; // flag in model_info that pre/postprocessing embedded size_t netInputHeight = 0; size_t netInputWidth = 0; diff --git a/model_api/cpp/models/include/models/instance_segmentation.h b/model_api/cpp/models/include/models/instance_segmentation.h index ee6458b4..27994a0b 100644 --- a/model_api/cpp/models/include/models/instance_segmentation.h +++ b/model_api/cpp/models/include/models/instance_segmentation.h @@ -35,13 +35,17 @@ class MaskRCNNModel : public ImageModel { MaskRCNNModel(std::shared_ptr& model, const ov::AnyMap& configuration); MaskRCNNModel(std::shared_ptr& adapter, const ov::AnyMap& configuration = {}); - static std::unique_ptr create_model(const std::string& modelFile, const ov::AnyMap& configuration = {}, bool preload = true, const std::string& device = "AUTO"); + static std::unique_ptr create_model(const std::string& modelFile, + const ov::AnyMap& configuration = {}, + bool preload = true, + const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); std::unique_ptr postprocess(InferenceResult& infResult) override; virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch(const std::vector& inputImgs); + virtual std::vector> inferBatch( + const std::vector& inputImgs); static std::string ModelType; bool postprocess_semantic_masks = true; diff --git a/model_api/cpp/models/include/models/keypoint_detection.h b/model_api/cpp/models/include/models/keypoint_detection.h index edc89de8..be038c8b 100644 --- a/model_api/cpp/models/include/models/keypoint_detection.h +++ b/model_api/cpp/models/include/models/keypoint_detection.h @@ -34,18 +34,21 @@ class KeypointDetectionModel : public ImageModel { KeypointDetectionModel(std::shared_ptr& model, const ov::AnyMap& configuration); KeypointDetectionModel(std::shared_ptr& adapter, const ov::AnyMap& configuration = {}); - static std::unique_ptr create_model(const std::string& modelFile, const ov::AnyMap& configuration = {}, bool preload = true, const std::string& device = "AUTO"); + static std::unique_ptr create_model(const std::string& modelFile, + const ov::AnyMap& configuration = {}, + bool preload = true, + const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); std::unique_ptr postprocess(InferenceResult& infResult) override; virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch(const std::vector& inputImgs); + virtual std::vector> inferBatch( + const std::vector& inputImgs); static std::string ModelType; protected: - void prepareInputsOutputs(std::shared_ptr& model) override; void updateModelInfo() override; void init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority); diff --git a/model_api/cpp/models/include/models/model_base.h b/model_api/cpp/models/include/models/model_base.h index 9fac826b..65135ebe 100644 --- a/model_api/cpp/models/include/models/model_base.h +++ b/model_api/cpp/models/include/models/model_base.h @@ -15,17 +15,16 @@ */ #pragma once -#include +#include + #include +#include #include -#include -#include - #include - +#include #include #include -#include +#include struct InferenceResult; struct InputData; @@ -50,13 +49,15 @@ class ModelBase { virtual std::unique_ptr postprocess(InferenceResult& infResult) = 0; virtual std::unique_ptr infer(const InputData& inputData); virtual void inferAsync(const InputData& inputData, const ov::AnyMap& callback_args = {}); - virtual std::vector> inferBatch(const std::vector>& inputData); + virtual std::vector> inferBatch( + const std::vector>& inputData); virtual std::vector> inferBatch(const std::vector& inputData); virtual bool isReady(); virtual void awaitAll(); virtual void awaitAny(); - virtual void setCallback(std::function, const ov::AnyMap& callback_args)> callback); + virtual void setCallback( + std::function, const ov::AnyMap& callback_args)> callback); virtual size_t getNumAsyncExecutors() const; const std::vector& getoutputNames() const { diff --git a/model_api/cpp/models/include/models/results.h b/model_api/cpp/models/include/models/results.h index 3090e301..3e325f2f 100644 --- a/model_api/cpp/models/include/models/results.h +++ b/model_api/cpp/models/include/models/results.h @@ -17,12 +17,11 @@ #pragma once #include #include -#include -#include - #include #include #include +#include +#include #include "internal_model_data.h" @@ -51,9 +50,8 @@ struct ResultBase { } }; -struct AnomalyResult : public ResultBase -{ - AnomalyResult(int64_t frameId = -1, const std::shared_ptr &metaData = nullptr) +struct AnomalyResult : public ResultBase { + AnomalyResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) : ResultBase(frameId, metaData) {} cv::Mat anomaly_map; std::vector pred_boxes; @@ -61,8 +59,7 @@ struct AnomalyResult : public ResultBase cv::Mat pred_mask; double pred_score; - friend std::ostream &operator<<(std::ostream &os, const AnomalyResult &prediction) - { + friend std::ostream& operator<<(std::ostream& os, const AnomalyResult& prediction) { double min_anomaly_map, max_anomaly_map; cv::minMaxLoc(prediction.anomaly_map, &min_anomaly_map, &max_anomaly_map); double min_pred_mask, max_pred_mask; @@ -70,21 +67,19 @@ struct AnomalyResult : public ResultBase os << "anomaly_map min:" << min_anomaly_map << " max:" << max_anomaly_map << ";"; os << "pred_score:" << std::fixed << std::setprecision(1) << prediction.pred_score << ";"; os << "pred_label:" << prediction.pred_label << ";"; - os << std::fixed << std::setprecision(0) << "pred_mask min:" << min_pred_mask << " max:" << max_pred_mask << ";"; + os << std::fixed << std::setprecision(0) << "pred_mask min:" << min_pred_mask << " max:" << max_pred_mask + << ";"; - if (!prediction.pred_boxes.empty()) - { + if (!prediction.pred_boxes.empty()) { os << "pred_boxes:"; - for (const cv::Rect &box : prediction.pred_boxes) - { + for (const cv::Rect& box : prediction.pred_boxes) { os << box << ","; } } return os; } - explicit operator std::string() - { + explicit operator std::string() { std::stringstream ss; ss << *this; return ss.str(); @@ -116,7 +111,7 @@ struct ClassificationResult : public ResultBase { ClassificationResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) : ResultBase(frameId, metaData) {} - friend std::ostream& operator<< (std::ostream& os, const ClassificationResult& prediction) { + friend std::ostream& operator<<(std::ostream& os, const ClassificationResult& prediction) { for (const ClassificationResult::Classification& classification : prediction.topLabels) { os << classification << ", "; } @@ -151,13 +146,15 @@ struct ClassificationResult : public ResultBase { Classification(unsigned int id, const std::string& label, float score) : id(id), label(label), score(score) {} - friend std::ostream& operator<< (std::ostream& os, const Classification& prediction) { - return os << prediction.id << " (" << prediction.label << "): " << std::fixed << std::setprecision(3) << prediction.score; + friend std::ostream& operator<<(std::ostream& os, const Classification& prediction) { + return os << prediction.id << " (" << prediction.label << "): " << std::fixed << std::setprecision(3) + << prediction.score; } }; std::vector topLabels; - ov::Tensor saliency_map, feature_vector, raw_scores; // Contains "raw_scores", "saliency_map" and "feature_vector" model outputs if such exist + ov::Tensor saliency_map, feature_vector, + raw_scores; // Contains "raw_scores", "saliency_map" and "feature_vector" model outputs if such exist }; struct DetectedObject : public cv::Rect2f { @@ -165,10 +162,10 @@ struct DetectedObject : public cv::Rect2f { std::string label; float confidence; - friend std::ostream& operator<< (std::ostream& os, const DetectedObject& detection) { - return os << int(detection.x) << ", " << int(detection.y) << ", " << int(detection.x + detection.width) - << ", " << int(detection.y + detection.height) << ", " - << detection.labelID << " (" << detection.label << "): " << std::fixed << std::setprecision(3) << detection.confidence; + friend std::ostream& operator<<(std::ostream& os, const DetectedObject& detection) { + return os << int(detection.x) << ", " << int(detection.y) << ", " << int(detection.x + detection.width) << ", " + << int(detection.y + detection.height) << ", " << detection.labelID << " (" << detection.label + << "): " << std::fixed << std::setprecision(3) << detection.confidence; } }; @@ -178,7 +175,7 @@ struct DetectionResult : public ResultBase { std::vector objects; ov::Tensor saliency_map, feature_vector; // Contan "saliency_map" and "feature_vector" model outputs if such exist - friend std::ostream& operator<< (std::ostream& os, const DetectionResult& prediction) { + friend std::ostream& operator<<(std::ostream& os, const DetectionResult& prediction) { for (const DetectedObject& obj : prediction.objects) { os << obj << "; "; } @@ -211,7 +208,7 @@ struct RetinaFaceDetectionResult : public DetectionResult { struct SegmentedObject : DetectedObject { cv::Mat mask; - friend std::ostream& operator<< (std::ostream& os, const SegmentedObject& prediction) { + friend std::ostream& operator<<(std::ostream& os, const SegmentedObject& prediction) { return os << static_cast(prediction) << ", " << cv::countNonZero(prediction.mask > 0.5); } }; @@ -221,10 +218,11 @@ struct SegmentedObjectWithRects : SegmentedObject { SegmentedObjectWithRects(const SegmentedObject& segmented_object) : SegmentedObject(segmented_object) {} - friend std::ostream& operator<< (std::ostream& os, const SegmentedObjectWithRects& prediction) { + friend std::ostream& operator<<(std::ostream& os, const SegmentedObjectWithRects& prediction) { os << static_cast(prediction) << std::fixed << std::setprecision(3); auto rect = prediction.rotated_rect; - os << ", RotatedRect: " << rect.center.x << ' ' << rect.center.y << ' ' << rect.size.width << ' ' << rect.size.height << ' ' << rect.angle; + os << ", RotatedRect: " << rect.center.x << ' ' << rect.center.y << ' ' << rect.size.width << ' ' + << rect.size.height << ' ' << rect.angle; return os; } }; @@ -263,16 +261,16 @@ struct ImageResult : public ResultBase { ImageResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) : ResultBase(frameId, metaData) {} cv::Mat resultImage; - friend std::ostream& operator<< (std::ostream& os, const ImageResult& prediction) { + friend std::ostream& operator<<(std::ostream& os, const ImageResult& prediction) { cv::Mat predicted_mask[] = {prediction.resultImage}; int nimages = 1; - int *channels = nullptr; + int* channels = nullptr; cv::Mat mask; cv::Mat outHist; int dims = 1; int histSize[] = {256}; float range[] = {0, 256}; - const float *ranges[] = {range}; + const float* ranges[] = {range}; cv::calcHist(predicted_mask, nimages, channels, mask, outHist, dims, histSize, ranges); os << std::fixed << std::setprecision(3); @@ -298,7 +296,7 @@ struct ImageResultWithSoftPrediction : public ImageResult { // Contain per class saliency_maps and "feature_vector" model output if feature_vector exists cv::Mat saliency_map; // Requires return_soft_prediction==true ov::Tensor feature_vector; - friend std::ostream& operator<< (std::ostream& os, const ImageResultWithSoftPrediction& prediction) { + friend std::ostream& operator<<(std::ostream& os, const ImageResultWithSoftPrediction& prediction) { os << static_cast(prediction) << '['; for (int i = 0; i < prediction.soft_prediction.dims; ++i) { os << prediction.soft_prediction.size[i] << ','; @@ -326,8 +324,9 @@ struct Contour { float probability; std::vector shape; - friend std::ostream& operator<< (std::ostream& os, const Contour& contour) { - return os << contour.label << ": " << std::fixed << std::setprecision(3) << contour.probability << ", " << contour.shape.size(); + friend std::ostream& operator<<(std::ostream& os, const Contour& contour) { + return os << contour.label << ": " << std::fixed << std::setprecision(3) << contour.probability << ", " + << contour.shape.size(); } }; @@ -335,8 +334,7 @@ static inline std::vector getContours(const std::vector combined_contours; std::vector> contours; for (const SegmentedObject& obj : segmentedObjects) { - cv::findContours(obj.mask, contours, cv::RETR_EXTERNAL, - cv::CHAIN_APPROX_NONE); + cv::findContours(obj.mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_NONE); // Assuming one contour output for findContours. Based on OTX this is a safe // assumption if (contours.size() != 1) { @@ -362,7 +360,7 @@ struct DetectedKeypoints { std::vector keypoints; std::vector scores; - friend std::ostream& operator<< (std::ostream& os, const DetectedKeypoints& prediction) { + friend std::ostream& operator<<(std::ostream& os, const DetectedKeypoints& prediction) { float kp_x_sum = 0.f; for (const cv::Point2f& keypoint : prediction.keypoints) { kp_x_sum += keypoint.x; @@ -372,8 +370,7 @@ struct DetectedKeypoints { return os; } - explicit operator std::string() - { + explicit operator std::string() { std::stringstream ss; ss << *this; return ss.str(); diff --git a/model_api/cpp/models/include/models/segmentation_model.h b/model_api/cpp/models/include/models/segmentation_model.h index 999cc838..aebd0c24 100644 --- a/model_api/cpp/models/include/models/segmentation_model.h +++ b/model_api/cpp/models/include/models/segmentation_model.h @@ -36,7 +36,10 @@ class SegmentationModel : public ImageModel { SegmentationModel(std::shared_ptr& model, const ov::AnyMap& configuration); SegmentationModel(std::shared_ptr& adapter, const ov::AnyMap& configuration = {}); - static std::unique_ptr create_model(const std::string& modelFile, const ov::AnyMap& configuration = {}, bool preload = true, const std::string& device = "AUTO"); + static std::unique_ptr create_model(const std::string& modelFile, + const ov::AnyMap& configuration = {}, + bool preload = true, + const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); std::unique_ptr postprocess(InferenceResult& infResult) override; @@ -48,7 +51,6 @@ class SegmentationModel : public ImageModel { std::vector getContours(const ImageResultWithSoftPrediction& imageResult); protected: - void prepareInputsOutputs(std::shared_ptr& model) override; void updateModelInfo() override; void init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority); @@ -58,4 +60,6 @@ class SegmentationModel : public ImageModel { bool return_soft_prediction = true; }; -cv::Mat create_hard_prediction_from_soft_prediction(const cv::Mat& soft_prediction, float soft_threshold, int blur_strength); +cv::Mat create_hard_prediction_from_soft_prediction(const cv::Mat& soft_prediction, + float soft_threshold, + int blur_strength); diff --git a/model_api/cpp/models/src/anomaly_model.cpp b/model_api/cpp/models/src/anomaly_model.cpp index 7d03ffab..4eb1e82b 100644 --- a/model_api/cpp/models/src/anomaly_model.cpp +++ b/model_api/cpp/models/src/anomaly_model.cpp @@ -45,7 +45,7 @@ AnomalyModel::AnomalyModel(std::shared_ptr& model, const ov::AnyMap& } AnomalyModel::AnomalyModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : ImageModel(adapter, configuration) { + : ImageModel(adapter, configuration) { init_from_config(configuration, adapter->getModelConfig()); } diff --git a/model_api/cpp/models/src/classification_model.cpp b/model_api/cpp/models/src/classification_model.cpp index 54695972..d22e99ba 100644 --- a/model_api/cpp/models/src/classification_model.cpp +++ b/model_api/cpp/models/src/classification_model.cpp @@ -21,21 +21,18 @@ #include #include #include -#include -#include -#include -#include - +#include #include #include #include - -#include - +#include +#include +#include #include +#include -#include "models/results.h" #include "models/input_data.h" +#include "models/results.h" std::string ClassificationModel::ModelType = "Classification"; @@ -86,8 +83,7 @@ void addOrFindSoftmaxAndTopkOutputs(std::shared_ptr& model, size_t to auto output_node = model->get_output_op(i)->input(0).get_source_output().get_node_shared_ptr(); if (std::string(output_node->get_type_name()) == "Softmax") { softmaxNode = output_node; - } - else if (std::string(output_node->get_type_name()) == "TopK") { + } else if (std::string(output_node->get_type_name()) == "TopK") { return; } } @@ -110,8 +106,7 @@ void addOrFindSoftmaxAndTopkOutputs(std::shared_ptr& model, size_t to if (add_raw_scores) { auto raw_scores = softmaxNode->output(0); outputs_vector = {scores, indices, raw_scores}; - } - else { + } else { outputs_vector = {scores, indices}; } for (const ov::Output& output : model->outputs()) { @@ -120,7 +115,8 @@ void addOrFindSoftmaxAndTopkOutputs(std::shared_ptr& model, size_t to } } - auto source_rt_info = model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}; + auto source_rt_info = + model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}; model = std::make_shared(outputs_vector, model->get_parameters(), "classification"); // preserve extra model_info items @@ -151,7 +147,8 @@ std::vector get_non_xai_names(const std::vector& output : outputs) { if (output.get_names().count(saliency_map_name) > 0) { continue; - } if (output.get_names().count(feature_vector_name) > 0) { + } + if (output.get_names().count(feature_vector_name) > 0) { continue; } outputNames.push_back(output.get_any_name()); @@ -164,8 +161,8 @@ std::vector get_non_xai_output_indices(const std::vector& output : outputs) { - bool is_xai = output.get_names().count(saliency_map_name) > 0 - || output.get_names().count(feature_vector_name) > 0; + bool is_xai = + output.get_names().count(saliency_map_name) > 0 || output.get_names().count(feature_vector_name) > 0; if (!is_xai) { outputIndices.push_back(idx); } @@ -180,7 +177,8 @@ std::vector get_non_xai_names(const std::vector& outpu for (const auto& output : outputs) { if (output.find(saliency_map_name) != std::string::npos) { continue; - } if (output.find(feature_vector_name) != std::string::npos) { + } + if (output.find(feature_vector_name) != std::string::npos) { continue; } outputNames.push_back(output); @@ -207,7 +205,7 @@ void append_xai_names(const std::vector& outputs, std::vector(hierarchical_info); - } - else if (hierarchical_postproc == "greedy") { + } else if (hierarchical_postproc == "greedy") { resolver = std::make_unique(hierarchical_info); - } - else { + } else { throw std::runtime_error("Wrong hierarchical labels postprocessing type"); } } } ClassificationModel::ClassificationModel(std::shared_ptr& model, const ov::AnyMap& configuration) - : ImageModel(model, configuration) { - init_from_config(configuration, model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); + : ImageModel(model, configuration) { + init_from_config(configuration, + model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); } ClassificationModel::ClassificationModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : ImageModel(adapter, configuration) { + : ImageModel(adapter, configuration) { outputNames = get_non_xai_names(adapter->getOutputNames()); append_xai_names(adapter->getOutputNames(), outputNames); init_from_config(configuration, adapter->getModelConfig()); @@ -259,7 +257,10 @@ void ClassificationModel::updateModelInfo() { model->set_rt_info(hierarchical_postproc, "model_info", "hierarchical_postproc"); } -std::unique_ptr ClassificationModel::create_model(const std::string& modelFile, const ov::AnyMap& configuration, bool preload, const std::string& device) { +std::unique_ptr ClassificationModel::create_model(const std::string& modelFile, + const ov::AnyMap& configuration, + bool preload, + const std::string& device) { auto core = ov::Core(); std::shared_ptr model = core.read_model(modelFile); @@ -270,11 +271,13 @@ std::unique_ptr ClassificationModel::create_model(const std model_type = model->get_rt_info("model_info", "model_type"); } } catch (const std::exception&) { - slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type << slog::endl; + slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type + << slog::endl; } if (model_type != ClassificationModel::ModelType) { - throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + model_type); + throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + + model_type); } std::unique_ptr classifier{new ClassificationModel(model, configuration)}; @@ -324,7 +327,8 @@ std::unique_ptr ClassificationModel::postprocess(InferenceResult& in return result; } -std::unique_ptr ClassificationModel::get_multilabel_predictions(InferenceResult& infResult, bool add_raw_scores) { +std::unique_ptr ClassificationModel::get_multilabel_predictions(InferenceResult& infResult, + bool add_raw_scores) { const ov::Tensor& logitsTensor = infResult.outputsData.find(outputNames[0])->second; const float* logitsPtr = logitsTensor.data(); @@ -353,7 +357,8 @@ std::unique_ptr ClassificationModel::get_multilabel_predictions(Infe return retVal; } -std::unique_ptr ClassificationModel::get_hierarchical_predictions(InferenceResult& infResult, bool add_raw_scores) { +std::unique_ptr ClassificationModel::get_hierarchical_predictions(InferenceResult& infResult, + bool add_raw_scores) { ClassificationResult* result = new ClassificationResult(infResult.frameId, infResult.metaData); const ov::Tensor& logitsTensor = infResult.outputsData.find(outputNames[0])->second; @@ -421,18 +426,19 @@ ov::Tensor ClassificationModel::reorder_saliency_maps(const ov::Tensor& source_m std::uint8_t* reordered_maps_ptr = static_cast(reordered_maps.data()); size_t shape_offset = (source_maps.get_shape().size() == 4) ? 1 : 0; - size_t map_byte_size = source_maps.get_element_type().size() * - source_maps.get_shape()[shape_offset + 1] * source_maps.get_shape()[shape_offset + 2]; + size_t map_byte_size = source_maps.get_element_type().size() * source_maps.get_shape()[shape_offset + 1] * + source_maps.get_shape()[shape_offset + 2]; for (size_t i = 0; i < source_maps.get_shape()[shape_offset]; ++i) { size_t new_index = hierarchical_info.label_to_idx[hierarchical_info.logit_idx_to_label[i]]; - std::copy_n(source_maps_ptr + i*map_byte_size, map_byte_size, reordered_maps_ptr + new_index * map_byte_size); + std::copy_n(source_maps_ptr + i * map_byte_size, map_byte_size, reordered_maps_ptr + new_index * map_byte_size); } return reordered_maps; } -std::unique_ptr ClassificationModel::get_multiclass_predictions(InferenceResult& infResult, bool add_raw_scores) { +std::unique_ptr ClassificationModel::get_multiclass_predictions(InferenceResult& infResult, + bool add_raw_scores) { const ov::Tensor& indicesTensor = infResult.outputsData.find(indices_name)->second; const int* indicesPtr = indicesTensor.data(); const ov::Tensor& scoresTensor = infResult.outputsData.find(scores_name)->second; @@ -473,21 +479,21 @@ void ClassificationModel::prepareInputsOutputs(std::shared_ptr& model const ov::Layout& inputLayout = getInputLayout(input); if (!embedded_processing) { - model = ImageModel::embedProcessing(model, - inputNames[0], - inputLayout, - resizeMode, - interpolationMode, - ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], - inputShape[ov::layout::height_idx(inputLayout)]}, - pad_value, - reverse_input_channels, - mean_values, - scale_values); + model = ImageModel::embedProcessing( + model, + inputNames[0], + inputLayout, + resizeMode, + interpolationMode, + ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], inputShape[ov::layout::height_idx(inputLayout)]}, + pad_value, + reverse_input_channels, + mean_values, + scale_values); ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); model = ppp.build(); - useAutoResize = true; // temporal solution for classification + useAutoResize = true; // temporal solution for classification } // --------------------------- Prepare output ----------------------------------------------------- @@ -500,24 +506,24 @@ void ClassificationModel::prepareInputsOutputs(std::shared_ptr& model const ov::Shape& outputShape = model->outputs()[non_xai_idx[0]].get_partial_shape().get_max_shape(); if (outputShape.size() != 2 && outputShape.size() != 4) { throw std::logic_error("Classification model wrapper supports topologies only with" - " 2-dimensional or 4-dimensional output"); + " 2-dimensional or 4-dimensional output"); } const ov::Layout outputLayout("NCHW"); if (outputShape.size() == 4 && (outputShape[ov::layout::height_idx(outputLayout)] != 1 || outputShape[ov::layout::width_idx(outputLayout)] != 1)) { throw std::logic_error("Classification model wrapper supports topologies only" - " with 4-dimensional output which has last two dimensions of size 1"); + " with 4-dimensional output which has last two dimensions of size 1"); } size_t classesNum = outputShape[ov::layout::channels_idx(outputLayout)]; if (topk > classesNum) { throw std::logic_error("The model provides " + std::to_string(classesNum) + " classes, but " + - std::to_string(topk) + " labels are requested to be predicted"); + std::to_string(topk) + " labels are requested to be predicted"); } if (classesNum != labels.size()) { throw std::logic_error("Model's number of classes and parsed labels must match (" + - std::to_string(outputShape[1]) + " and " + std::to_string(labels.size()) + ')'); + std::to_string(outputShape[1]) + " and " + std::to_string(labels.size()) + ')'); } } @@ -545,7 +551,8 @@ std::unique_ptr ClassificationModel::infer(const ImageInpu return std::unique_ptr(static_cast(result.release())); } -std::vector> ClassificationModel::inferBatch(const std::vector& inputImgs) { +std::vector> ClassificationModel::inferBatch( + const std::vector& inputImgs) { auto results = ImageModel::inferBatchImage(inputImgs); std::vector> clsResults; clsResults.reserve(results.size()); @@ -555,7 +562,7 @@ std::vector> ClassificationModel::inferBat return clsResults; } -HierarchicalConfig::HierarchicalConfig(const std::string& json_repr) { +HierarchicalConfig::HierarchicalConfig(const std::string& json_repr) { nlohmann::json data = nlohmann::json::parse(json_repr); num_multilabel_heads = data.at("cls_heads_info").at("num_multilabel_classes"); @@ -566,7 +573,7 @@ HierarchicalConfig::HierarchicalConfig(const std::string& json_repr) { data.at("cls_heads_info").at("all_groups").get_to(all_groups); data.at("label_tree_edges").get_to(label_tree_edges); - std::map> tmp_head_idx_to_logits_range; + std::map> tmp_head_idx_to_logits_range; data.at("cls_heads_info").at("head_idx_to_logits_range").get_to(tmp_head_idx_to_logits_range); for (const auto& range_descr : tmp_head_idx_to_logits_range) { @@ -585,13 +592,14 @@ HierarchicalConfig::HierarchicalConfig(const std::string& json_repr) { } } -GreedyLabelsResolver::GreedyLabelsResolver(const HierarchicalConfig& config) : - label_to_idx(config.label_to_idx), - label_relations(config.label_tree_edges), - label_groups(config.all_groups) {} +GreedyLabelsResolver::GreedyLabelsResolver(const HierarchicalConfig& config) + : label_to_idx(config.label_to_idx), + label_relations(config.label_tree_edges), + label_groups(config.all_groups) {} -std::map GreedyLabelsResolver::resolve_labels(const std::vector>& labels, - const std::vector& scores) { +std::map GreedyLabelsResolver::resolve_labels( + const std::vector>& labels, + const std::vector& scores) { if (labels.size() != scores.size()) { throw std::runtime_error("Inconsistent number of labels and scores"); } @@ -608,8 +616,7 @@ std::map GreedyLabelsResolver::resolve_labels(const std::vec for (const auto& g : label_groups) { if (g.size() == 1 && label_to_prob[g[0]] > 0.f) { candidates.push_back(g[0]); - } - else { + } else { float max_prob = 0.f; std::string max_label; for (const auto& lbl : g) { @@ -649,7 +656,8 @@ std::string GreedyLabelsResolver::get_parent(const std::string& label) { return ""; } -std::vector GreedyLabelsResolver::get_predecessors(const std::string& label, const std::vector& candidates) { +std::vector GreedyLabelsResolver::get_predecessors(const std::string& label, + const std::vector& candidates) { std::vector predecessors; auto last_parent = get_parent(label); @@ -662,7 +670,6 @@ std::vector GreedyLabelsResolver::get_predecessors(const std::strin } predecessors.push_back(last_parent); last_parent = get_parent(last_parent); - } if (predecessors.size() > 0) { @@ -672,10 +679,9 @@ std::vector GreedyLabelsResolver::get_predecessors(const std::strin return predecessors; } - SimpleLabelsGraph::SimpleLabelsGraph(const std::vector& vertices_) - : vertices(vertices_), t_sort_cache_valid(false) { -} + : vertices(vertices_), + t_sort_cache_valid(false) {} void SimpleLabelsGraph::add_edge(const std::string& parent, const std::string& child) { adj[parent].push_back(child); @@ -762,7 +768,6 @@ std::vector SimpleLabelsGraph::topological_sort() { return ordered_nodes; } - ProbabilisticLabelsResolver::ProbabilisticLabelsResolver(const HierarchicalConfig& conf) : GreedyLabelsResolver(conf) { std::vector all_labels; for (const auto& item : label_to_idx) { @@ -775,8 +780,9 @@ ProbabilisticLabelsResolver::ProbabilisticLabelsResolver(const HierarchicalConfi label_tree.get_labels_in_topological_order(); } -std::map ProbabilisticLabelsResolver::resolve_labels(const std::vector>& labels, - const std::vector& scores) { +std::map ProbabilisticLabelsResolver::resolve_labels( + const std::vector>& labels, + const std::vector& scores) { if (labels.size() != scores.size()) { throw std::runtime_error("Inconsistent number of labels and scores"); } @@ -796,8 +802,7 @@ std::map ProbabilisticLabelsResolver::resolve_labels(const s if (item.second > 0) { if (label_to_prob.find(item.first) != label_to_prob.end()) { output_labels_map[item.first] = item.second * label_to_prob[item.first]; - } - else { + } else { output_labels_map[item.first] = item.second; } } @@ -806,7 +811,8 @@ std::map ProbabilisticLabelsResolver::resolve_labels(const s return output_labels_map; } -std::unordered_map ProbabilisticLabelsResolver::add_missing_ancestors(const std::unordered_map& label_to_prob) const { +std::unordered_map ProbabilisticLabelsResolver::add_missing_ancestors( + const std::unordered_map& label_to_prob) const { std::unordered_map updated_label_to_probability(label_to_prob); for (const auto& item : label_to_prob) { for (const auto& ancestor : label_tree.get_ancestors(item.first)) { @@ -818,7 +824,8 @@ std::unordered_map ProbabilisticLabelsResolver::add_missing_ return updated_label_to_probability; } -std::map ProbabilisticLabelsResolver::resolve_exclusive_labels(const std::unordered_map& label_to_prob) const { +std::map ProbabilisticLabelsResolver::resolve_exclusive_labels( + const std::unordered_map& label_to_prob) const { std::map hard_classification; for (const auto& item : label_to_prob) { diff --git a/model_api/cpp/models/src/detection_model.cpp b/model_api/cpp/models/src/detection_model.cpp index efb0c23b..858e77aa 100644 --- a/model_api/cpp/models/src/detection_model.cpp +++ b/model_api/cpp/models/src/detection_model.cpp @@ -15,16 +15,16 @@ */ #include "models/detection_model.h" -#include "models/detection_model_ssd.h" -#include "models/detection_model_yolo.h" -#include "models/detection_model_yolov3_onnx.h" -#include "models/detection_model_yolox.h" #include #include #include #include +#include "models/detection_model_ssd.h" +#include "models/detection_model_yolo.h" +#include "models/detection_model_yolov3_onnx.h" +#include "models/detection_model_yolox.h" #include "models/image_model.h" #include "models/input_data.h" #include "models/results.h" @@ -43,8 +43,9 @@ DetectionModel::DetectionModel(std::shared_ptr& model, const ov::AnyM } DetectionModel::DetectionModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : ImageModel(adapter, configuration) { - confidence_threshold = get_from_any_maps("confidence_threshold", configuration, adapter->getModelConfig(), confidence_threshold); + : ImageModel(adapter, configuration) { + confidence_threshold = + get_from_any_maps("confidence_threshold", configuration, adapter->getModelConfig(), confidence_threshold); } void DetectionModel::updateModelInfo() { @@ -62,11 +63,12 @@ std::unique_ptr DetectionModel::create_model(const std::string& std::shared_ptr model = core.read_model(modelFile); if (model_type.empty()) { try { - if (model->has_rt_info("model_info", "model_type") ) { + if (model->has_rt_info("model_info", "model_type")) { model_type = model->get_rt_info("model_info", "model_type"); } } catch (const std::exception&) { - slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type << slog::endl; + slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type + << slog::endl; } } @@ -80,7 +82,8 @@ std::unique_ptr DetectionModel::create_model(const std::string& } else if (model_type == YOLOv8::ModelType) { detectionModel = std::unique_ptr(new YOLOv8(model, configuration)); } else { - throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + model_type); + throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + + model_type); } detectionModel->prepare(); diff --git a/model_api/cpp/models/src/detection_model_ext.cpp b/model_api/cpp/models/src/detection_model_ext.cpp index 33b10091..d1d7792a 100644 --- a/model_api/cpp/models/src/detection_model_ext.cpp +++ b/model_api/cpp/models/src/detection_model_ext.cpp @@ -21,8 +21,7 @@ DetectionModelExt::DetectionModelExt(std::shared_ptr& model, const ov } } -DetectionModelExt::DetectionModelExt(std::shared_ptr& adapter) - : DetectionModel(adapter) { +DetectionModelExt::DetectionModelExt(std::shared_ptr& adapter) : DetectionModel(adapter) { const ov::AnyMap& configuration = adapter->getModelConfig(); auto iou_threshold_iter = configuration.find("iou_threshold"); if (iou_threshold_iter != configuration.end()) { diff --git a/model_api/cpp/models/src/detection_model_ssd.cpp b/model_api/cpp/models/src/detection_model_ssd.cpp index 2afe9e26..815ca01f 100644 --- a/model_api/cpp/models/src/detection_model_ssd.cpp +++ b/model_api/cpp/models/src/detection_model_ssd.cpp @@ -18,15 +18,13 @@ #include #include +#include #include #include #include -#include - -#include - #include #include +#include #include "models/internal_model_data.h" #include "models/results.h" @@ -76,20 +74,21 @@ NumAndStep fromMultipleOutputs(const ov::Shape& boxesShape) { return {detectionsNum, objectSize}; } throw std::logic_error("Incorrect number of 'boxes' output dimensions, expected 2 or 3, but had " + - std::to_string(boxesShape.size())); + std::to_string(boxesShape.size())); } std::vector filterOutXai(const std::vector& names) { std::vector filtered; - std::copy_if (names.begin(), names.end(), std::back_inserter(filtered), [](const std::string& name){return name != saliency_map_name && name != feature_vector_name;}); + std::copy_if(names.begin(), names.end(), std::back_inserter(filtered), [](const std::string& name) { + return name != saliency_map_name && name != feature_vector_name; + }); return filtered; } - float clamp_and_round(float val, float min, float max) { return std::round(std::max(min, std::min(max, val))); } -} +} // namespace std::string ModelSSD::ModelType = "ssd"; @@ -106,7 +105,8 @@ std::shared_ptr ModelSSD::preprocess(const InputData& inputDa } std::unique_ptr ModelSSD::postprocess(InferenceResult& infResult) { - std::unique_ptr result = filterOutXai(outputNames).size() > 1 ? postprocessMultipleOutputs(infResult) : postprocessSingleOutput(infResult); + std::unique_ptr result = filterOutXai(outputNames).size() > 1 ? postprocessMultipleOutputs(infResult) + : postprocessSingleOutput(infResult); DetectionResult* cls_res = static_cast(result.get()); auto saliency_map_iter = infResult.outputsData.find(saliency_map_name); if (saliency_map_iter != infResult.outputsData.end()) { @@ -131,9 +131,8 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i const auto& internalData = infResult.internalModelData->asRef(); float floatInputImgWidth = float(internalData.inputImgWidth), - floatInputImgHeight = float(internalData.inputImgHeight); - float invertedScaleX = floatInputImgWidth / netInputWidth, - invertedScaleY = floatInputImgHeight / netInputHeight; + floatInputImgHeight = float(internalData.inputImgHeight); + float invertedScaleX = floatInputImgWidth / netInputWidth, invertedScaleY = floatInputImgHeight / netInputHeight; int padLeft = 0, padTop = 0; if (RESIZE_KEEP_ASPECT == resizeMode || RESIZE_KEEP_ASPECT_LETTERBOX == resizeMode) { invertedScaleX = invertedScaleY = std::max(invertedScaleX, invertedScaleY); @@ -158,22 +157,24 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i desc.confidence = confidence; desc.labelID = static_cast(detections[i * numAndStep.objectSize + 1]); desc.label = getLabelName(desc.labelID); - desc.x = clamp( - round((detections[i * numAndStep.objectSize + 3] * netInputWidth - padLeft) * invertedScaleX), - 0.f, - floatInputImgWidth); - desc.y = clamp( - round((detections[i * numAndStep.objectSize + 4] * netInputHeight - padTop) * invertedScaleY), - 0.f, - floatInputImgHeight); - desc.width = clamp( - round((detections[i * numAndStep.objectSize + 5] * netInputWidth - padLeft) * invertedScaleX), - 0.f, - floatInputImgWidth) - desc.x; - desc.height = clamp( - round((detections[i * numAndStep.objectSize + 6] * netInputHeight - padTop) * invertedScaleY), - 0.f, - floatInputImgHeight) - desc.y; + desc.x = + clamp(round((detections[i * numAndStep.objectSize + 3] * netInputWidth - padLeft) * invertedScaleX), + 0.f, + floatInputImgWidth); + desc.y = + clamp(round((detections[i * numAndStep.objectSize + 4] * netInputHeight - padTop) * invertedScaleY), + 0.f, + floatInputImgHeight); + desc.width = + clamp(round((detections[i * numAndStep.objectSize + 5] * netInputWidth - padLeft) * invertedScaleX), + 0.f, + floatInputImgWidth) - + desc.x; + desc.height = + clamp(round((detections[i * numAndStep.objectSize + 6] * netInputHeight - padTop) * invertedScaleY), + 0.f, + floatInputImgHeight) - + desc.y; result->objects.push_back(desc); } } @@ -186,16 +187,16 @@ std::unique_ptr ModelSSD::postprocessMultipleOutputs(InferenceResult const float* boxes = infResult.outputsData[namesWithoutXai[0]].data(); NumAndStep numAndStep = fromMultipleOutputs(infResult.outputsData[namesWithoutXai[0]].get_shape()); const int64_t* labels = infResult.outputsData[namesWithoutXai[1]].data(); - const float* scores = namesWithoutXai.size() > 2 ? infResult.outputsData[namesWithoutXai[2]].data() : nullptr; + const float* scores = + namesWithoutXai.size() > 2 ? infResult.outputsData[namesWithoutXai[2]].data() : nullptr; DetectionResult* result = new DetectionResult(infResult.frameId, infResult.metaData); auto retVal = std::unique_ptr(result); const auto& internalData = infResult.internalModelData->asRef(); float floatInputImgWidth = float(internalData.inputImgWidth), - floatInputImgHeight = float(internalData.inputImgHeight); - float invertedScaleX = floatInputImgWidth / netInputWidth, - invertedScaleY = floatInputImgHeight / netInputHeight; + floatInputImgHeight = float(internalData.inputImgHeight); + float invertedScaleX = floatInputImgWidth / netInputWidth, invertedScaleY = floatInputImgHeight / netInputHeight; int padLeft = 0, padTop = 0; if (RESIZE_KEEP_ASPECT == resizeMode || RESIZE_KEEP_ASPECT_LETTERBOX == resizeMode) { invertedScaleX = invertedScaleY = std::max(invertedScaleX, invertedScaleY); @@ -220,10 +221,21 @@ std::unique_ptr ModelSSD::postprocessMultipleOutputs(InferenceResult desc.confidence = confidence; desc.labelID = labels[i]; desc.label = getLabelName(desc.labelID); - desc.x = clamp_and_round((boxes[i * numAndStep.objectSize] * widthScale - padLeft) * invertedScaleX, 0.f, floatInputImgWidth); - desc.y = clamp_and_round((boxes[i * numAndStep.objectSize + 1] * heightScale - padTop) * invertedScaleY, 0.f, floatInputImgHeight); - desc.width = clamp_and_round((boxes[i * numAndStep.objectSize + 2] * widthScale - padLeft) * invertedScaleX, 0.f, floatInputImgWidth) - desc.x; - desc.height = clamp_and_round((boxes[i * numAndStep.objectSize + 3] * heightScale - padTop) * invertedScaleY, 0.f, floatInputImgHeight) - desc.y; + desc.x = clamp_and_round((boxes[i * numAndStep.objectSize] * widthScale - padLeft) * invertedScaleX, + 0.f, + floatInputImgWidth); + desc.y = clamp_and_round((boxes[i * numAndStep.objectSize + 1] * heightScale - padTop) * invertedScaleY, + 0.f, + floatInputImgHeight); + desc.width = clamp_and_round((boxes[i * numAndStep.objectSize + 2] * widthScale - padLeft) * invertedScaleX, + 0.f, + floatInputImgWidth) - + desc.x; + desc.height = + clamp_and_round((boxes[i * numAndStep.objectSize + 3] * heightScale - padTop) * invertedScaleY, + 0.f, + floatInputImgHeight) - + desc.y; if (desc.width * desc.height >= box_area_threshold) { result->objects.push_back(desc); @@ -250,22 +262,22 @@ void ModelSSD::prepareInputsOutputs(std::shared_ptr& model) { } if (!embedded_processing) { - model = ImageModel::embedProcessing(model, - inputNames[0], - inputLayout, - resizeMode, - interpolationMode, - ov::Shape{shape[ov::layout::width_idx(inputLayout)], - shape[ov::layout::height_idx(inputLayout)]}, - pad_value, - reverse_input_channels, - mean_values, - scale_values); + model = ImageModel::embedProcessing( + model, + inputNames[0], + inputLayout, + resizeMode, + interpolationMode, + ov::Shape{shape[ov::layout::width_idx(inputLayout)], shape[ov::layout::height_idx(inputLayout)]}, + pad_value, + reverse_input_channels, + mean_values, + scale_values); netInputWidth = shape[ov::layout::width_idx(inputLayout)]; netInputHeight = shape[ov::layout::height_idx(inputLayout)]; - useAutoResize = true; // temporal solution for SSD + useAutoResize = true; // temporal solution for SSD embedded_processing = true; } } else if (shape.size() == 2) { // 2nd input contains image info diff --git a/model_api/cpp/models/src/detection_model_yolo.cpp b/model_api/cpp/models/src/detection_model_yolo.cpp index 033be849..ccea2db2 100644 --- a/model_api/cpp/models/src/detection_model_yolo.cpp +++ b/model_api/cpp/models/src/detection_model_yolo.cpp @@ -19,16 +19,14 @@ #include #include #include +#include #include #include #include -#include - -#include - #include #include #include +#include #include "models/internal_model_data.h" #include "models/results.h" @@ -87,14 +85,15 @@ float sigmoid(float x) noexcept { constexpr float identity(float x) noexcept { return x; } -} +} // namespace ModelYolo::ModelYolo(std::shared_ptr& model, const ov::AnyMap& configuration) : DetectionModelExt(model, configuration) { auto anchors_iter = configuration.find("anchors"); if (anchors_iter == configuration.end()) { if (model->has_rt_info("model_info", "anchors")) { - //presetAnchors = model->get_rt_info().at("model_info").as>().get().at("anchors").as>(); + // presetAnchors = + // model->get_rt_info().at("model_info").as>().get().at("anchors").as>(); presetAnchors = model->get_rt_info>("model_info", "anchors"); } } else { @@ -109,11 +108,10 @@ ModelYolo::ModelYolo(std::shared_ptr& model, const ov::AnyMap& config presetMasks = masks_iter->second.as>(); } - resizeMode = RESIZE_FILL; // Ignore resize_type for now + resizeMode = RESIZE_FILL; // Ignore resize_type for now } -ModelYolo::ModelYolo(std::shared_ptr& adapter) - : DetectionModelExt(adapter) { +ModelYolo::ModelYolo(std::shared_ptr& adapter) : DetectionModelExt(adapter) { const ov::AnyMap& configuration = adapter->getModelConfig(); auto anchors_iter = configuration.find("anchors"); if (anchors_iter != configuration.end()) { @@ -124,7 +122,7 @@ ModelYolo::ModelYolo(std::shared_ptr& adapter) presetMasks = masks_iter->second.as>(); } - resizeMode = RESIZE_FILL; // Ignore resize_type for now + resizeMode = RESIZE_FILL; // Ignore resize_type for now } void ModelYolo::prepareInputsOutputs(std::shared_ptr& model) { @@ -207,37 +205,35 @@ void ModelYolo::prepareInputsOutputs(std::shared_ptr& model) { if (!isRegionFound) { switch (outputNames.size()) { - case 1: - yoloVersion = YoloVersion::YOLOF; - break; - case 2: - yoloVersion = YoloVersion::YOLO_V4_TINY; - break; - case 3: - yoloVersion = YoloVersion::YOLO_V4; - break; + case 1: + yoloVersion = YoloVersion::YOLOF; + break; + case 2: + yoloVersion = YoloVersion::YOLO_V4_TINY; + break; + case 3: + yoloVersion = YoloVersion::YOLO_V4; + break; } int num = yoloVersion == YoloVersion::YOLOF ? 6 : 3; isObjConf = yoloVersion == YoloVersion::YOLOF ? 0 : 1; int i = 0; - const std::vector defaultMasks[]{ - // YOLOv1v2 - {}, - // YOLOv3 - {}, - // YOLOv4 - {0, 1, 2, 3, 4, 5, 6, 7, 8}, - // YOLOv4_Tiny - {1, 2, 3, 3, 4, 5}, - // YOLOF - {0, 1, 2, 3, 4, 5}}; + const std::vector defaultMasks[]{// YOLOv1v2 + {}, + // YOLOv3 + {}, + // YOLOv4 + {0, 1, 2, 3, 4, 5, 6, 7, 8}, + // YOLOv4_Tiny + {1, 2, 3, 3, 4, 5}, + // YOLOF + {0, 1, 2, 3, 4, 5}}; auto chosenMasks = presetMasks.size() ? presetMasks : defaultMasks[size_t(yoloVersion)]; if (chosenMasks.size() != num * outputs.size()) { - throw std::runtime_error("Invalid size of masks array, got " + - std::to_string(presetMasks.size()) + ", should be " + - std::to_string(num * outputs.size())); + throw std::runtime_error("Invalid size of masks array, got " + std::to_string(presetMasks.size()) + + ", should be " + std::to_string(num * outputs.size())); } std::sort(outputNames.begin(), @@ -345,29 +341,32 @@ void ModelYolo::parseYOLOOutput(const std::string& output_name, unsigned long scaleH; unsigned long scaleW; switch (yoloVersion) { - case YoloVersion::YOLO_V1V2: - sideH = region.outputHeight; - sideW = region.outputWidth; - scaleW = region.outputWidth; - scaleH = region.outputHeight; - break; - case YoloVersion::YOLO_V3: - case YoloVersion::YOLO_V4: - case YoloVersion::YOLO_V4_TINY: - case YoloVersion::YOLOF: - sideH = static_cast(tensor.get_shape()[ov::layout::height_idx("NCHW")]); - sideW = static_cast(tensor.get_shape()[ov::layout::width_idx("NCHW")]); - scaleW = resized_im_w; - scaleH = resized_im_h; - break; - default: throw std::runtime_error("Unknown YoloVersion"); + case YoloVersion::YOLO_V1V2: + sideH = region.outputHeight; + sideW = region.outputWidth; + scaleW = region.outputWidth; + scaleH = region.outputHeight; + break; + case YoloVersion::YOLO_V3: + case YoloVersion::YOLO_V4: + case YoloVersion::YOLO_V4_TINY: + case YoloVersion::YOLOF: + sideH = static_cast(tensor.get_shape()[ov::layout::height_idx("NCHW")]); + sideW = static_cast(tensor.get_shape()[ov::layout::width_idx("NCHW")]); + scaleW = resized_im_w; + scaleH = resized_im_h; + break; + default: + throw std::runtime_error("Unknown YoloVersion"); } auto entriesNum = sideW * sideH; const float* outData = tensor.data(); - auto postprocessRawData = - (yoloVersion == YoloVersion::YOLO_V4 || yoloVersion == YoloVersion::YOLO_V4_TINY || yoloVersion == YoloVersion::YOLOF) ? sigmoid : identity; + auto postprocessRawData = (yoloVersion == YoloVersion::YOLO_V4 || yoloVersion == YoloVersion::YOLO_V4_TINY || + yoloVersion == YoloVersion::YOLOF) + ? sigmoid + : identity; // --------------------------- Parsing YOLO Region output ------------------------------------- for (int i = 0; i < entriesNum; ++i) { @@ -517,19 +516,17 @@ void YOLOv5::prepareInputsOutputs(std::shared_ptr& model) { inputNames.push_back(input.get_any_name()); const ov::Layout& inputLayout = getInputLayout(input); if (!embedded_processing) { - model = ImageModel::embedProcessing(model, - inputNames[0], - inputLayout, - resizeMode, - interpolationMode, - ov::Shape{ - in_shape[ov::layout::width_idx(inputLayout)], - in_shape[ov::layout::height_idx(inputLayout)] - }, - pad_value, - reverse_input_channels, - mean_values, - scale_values); + model = ImageModel::embedProcessing( + model, + inputNames[0], + inputLayout, + resizeMode, + interpolationMode, + ov::Shape{in_shape[ov::layout::width_idx(inputLayout)], in_shape[ov::layout::height_idx(inputLayout)]}, + pad_value, + reverse_input_channels, + mean_values, + scale_values); netInputWidth = in_shape[ov::layout::width_idx(inputLayout)]; netInputHeight = in_shape[ov::layout::height_idx(inputLayout)]; @@ -559,7 +556,8 @@ void YOLOv5::updateModelInfo() { void YOLOv5::init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority) { pad_value = get_from_any_maps("pad_value", top_priority, mid_priority, 114); - if (top_priority.find("resize_type") == top_priority.end() && mid_priority.find("resize_type") == mid_priority.end()) { + if (top_priority.find("resize_type") == top_priority.end() && + mid_priority.find("resize_type") == mid_priority.end()) { interpolationMode = cv::INTER_LINEAR; resizeMode = RESIZE_KEEP_ASPECT_LETTERBOX; } @@ -571,12 +569,11 @@ void YOLOv5::init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& } YOLOv5::YOLOv5(std::shared_ptr& model, const ov::AnyMap& configuration) - : DetectionModelExt(model, configuration) { + : DetectionModelExt(model, configuration) { init_from_config(configuration, model->get_rt_info("model_info")); } -YOLOv5::YOLOv5(std::shared_ptr& adapter) - : DetectionModelExt(adapter) { +YOLOv5::YOLOv5(std::shared_ptr& adapter) : DetectionModelExt(adapter) { init_from_config(adapter->getModelConfig(), ov::AnyMap{}); } @@ -607,13 +604,11 @@ std::unique_ptr YOLOv5::postprocess(InferenceResult& infResult) { } } if (confidence > confidence_threshold) { - boxes_with_class.emplace_back( - detections[0 * num_proposals + i] - detections[2 * num_proposals + i] / 2.0f, - detections[1 * num_proposals + i] - detections[3 * num_proposals + i] / 2.0f, - detections[0 * num_proposals + i] + detections[2 * num_proposals + i] / 2.0f, - detections[1 * num_proposals + i] + detections[3 * num_proposals + i] / 2.0f, - max_id - LABELS_START - ); + boxes_with_class.emplace_back(detections[0 * num_proposals + i] - detections[2 * num_proposals + i] / 2.0f, + detections[1 * num_proposals + i] - detections[3 * num_proposals + i] / 2.0f, + detections[0 * num_proposals + i] + detections[2 * num_proposals + i] / 2.0f, + detections[1 * num_proposals + i] + detections[3 * num_proposals + i] / 2.0f, + max_id - LABELS_START); confidences.push_back(confidence); } } @@ -629,9 +624,8 @@ std::unique_ptr YOLOv5::postprocess(InferenceResult& infResult) { auto base = std::unique_ptr(result); const auto& internalData = infResult.internalModelData->asRef(); float floatInputImgWidth = float(internalData.inputImgWidth), - floatInputImgHeight = float(internalData.inputImgHeight); - float invertedScaleX = floatInputImgWidth / netInputWidth, - invertedScaleY = floatInputImgHeight / netInputHeight; + floatInputImgHeight = float(internalData.inputImgHeight); + float invertedScaleX = floatInputImgWidth / netInputWidth, invertedScaleY = floatInputImgHeight / netInputHeight; int padLeft = 0, padTop = 0; if (RESIZE_KEEP_ASPECT == resizeMode || RESIZE_KEEP_ASPECT_LETTERBOX == resizeMode) { invertedScaleX = invertedScaleY = std::max(invertedScaleX, invertedScaleY); @@ -642,22 +636,12 @@ std::unique_ptr YOLOv5::postprocess(InferenceResult& infResult) { } for (size_t idx : keep) { DetectedObject desc; - desc.x = clamp( - round((boxes_with_class[idx].left - padLeft) * invertedScaleX), - 0.f, - floatInputImgWidth); - desc.y = clamp( - round((boxes_with_class[idx].top - padTop) * invertedScaleY), - 0.f, - floatInputImgHeight); - desc.width = clamp( - round((boxes_with_class[idx].right - padLeft) * invertedScaleX), - 0.f, - floatInputImgWidth) - desc.x; - desc.height = clamp( - round((boxes_with_class[idx].bottom - padTop) * invertedScaleY), - 0.f, - floatInputImgHeight) - desc.y; + desc.x = clamp(round((boxes_with_class[idx].left - padLeft) * invertedScaleX), 0.f, floatInputImgWidth); + desc.y = clamp(round((boxes_with_class[idx].top - padTop) * invertedScaleY), 0.f, floatInputImgHeight); + desc.width = + clamp(round((boxes_with_class[idx].right - padLeft) * invertedScaleX), 0.f, floatInputImgWidth) - desc.x; + desc.height = + clamp(round((boxes_with_class[idx].bottom - padTop) * invertedScaleY), 0.f, floatInputImgHeight) - desc.y; desc.confidence = confidences[idx]; desc.labelID = static_cast(boxes_with_class[idx].labelID); desc.label = getLabelName(desc.labelID); diff --git a/model_api/cpp/models/src/detection_model_yolov3_onnx.cpp b/model_api/cpp/models/src/detection_model_yolov3_onnx.cpp index cebd59f7..a454286b 100644 --- a/model_api/cpp/models/src/detection_model_yolov3_onnx.cpp +++ b/model_api/cpp/models/src/detection_model_yolov3_onnx.cpp @@ -19,24 +19,21 @@ #include #include #include +#include #include #include #include -#include - -#include - #include #include +#include #include "models/input_data.h" #include "models/internal_model_data.h" #include "models/results.h" #include "utils/image_utils.h" - void ModelYoloV3ONNX::initDefaultParameters(const ov::AnyMap&) { - resizeMode = RESIZE_KEEP_ASPECT_LETTERBOX; // Ignore configuration for now + resizeMode = RESIZE_KEEP_ASPECT_LETTERBOX; // Ignore configuration for now useAutoResize = false; } @@ -45,8 +42,7 @@ ModelYoloV3ONNX::ModelYoloV3ONNX(std::shared_ptr& model, const ov::An initDefaultParameters(configuration); } -ModelYoloV3ONNX::ModelYoloV3ONNX(std::shared_ptr& adapter) - : DetectionModel(adapter) { +ModelYoloV3ONNX::ModelYoloV3ONNX(std::shared_ptr& adapter) : DetectionModel(adapter) { const ov::AnyMap& configuration = adapter->getModelConfig(); initDefaultParameters(configuration); } @@ -103,16 +99,15 @@ void ModelYoloV3ONNX::prepareInputsOutputs(std::shared_ptr& model) { scoresOutputName = currentName; ppp.output(currentName).tensor().set_element_type(ov::element::f32); } else { - throw std::logic_error("Expected shapes [:,:,4], [:," - + std::to_string(numberOfClasses) + ",:] and [:,3] for outputs"); + throw std::logic_error("Expected shapes [:,:,4], [:," + std::to_string(numberOfClasses) + + ",:] and [:,3] for outputs"); } outputNames.push_back(currentName); } model = ppp.build(); } -std::shared_ptr ModelYoloV3ONNX::preprocess(const InputData& inputData, - InferenceInput& input) { +std::shared_ptr ModelYoloV3ONNX::preprocess(const InputData& inputData, InferenceInput& input) { const auto& origImg = inputData.asRef().inputImage; ov::Tensor info{ov::element::i32, ov::Shape({1, 2})}; int32_t* data = info.data(); @@ -130,7 +125,7 @@ float getScore(const ov::Tensor& scoresTensor, size_t classInd, size_t boxInd) { return scoresPtr[classInd * N + boxInd]; } -} +} // namespace std::unique_ptr ModelYoloV3ONNX::postprocess(InferenceResult& infResult) { // Get info about input image @@ -183,7 +178,6 @@ std::unique_ptr ModelYoloV3ONNX::postprocess(InferenceResult& infRes obj.label = getLabelName(classInd); result->objects.push_back(obj); - } } diff --git a/model_api/cpp/models/src/detection_model_yolox.cpp b/model_api/cpp/models/src/detection_model_yolox.cpp index be895195..5ba3d891 100644 --- a/model_api/cpp/models/src/detection_model_yolox.cpp +++ b/model_api/cpp/models/src/detection_model_yolox.cpp @@ -19,15 +19,13 @@ #include #include #include +#include #include #include #include -#include - -#include - #include #include +#include #include "models/input_data.h" #include "models/internal_model_data.h" @@ -38,7 +36,7 @@ std::string ModelYoloX::ModelType = "yolox"; void ModelYoloX::initDefaultParameters(const ov::AnyMap&) { - resizeMode = RESIZE_KEEP_ASPECT; // Ignore configuration for now + resizeMode = RESIZE_KEEP_ASPECT; // Ignore configuration for now useAutoResize = false; } @@ -47,8 +45,7 @@ ModelYoloX::ModelYoloX(std::shared_ptr& model, const ov::AnyMap& conf initDefaultParameters(configuration); } -ModelYoloX::ModelYoloX(std::shared_ptr& adapter) - : DetectionModelExt(adapter) { +ModelYoloX::ModelYoloX(std::shared_ptr& adapter) : DetectionModelExt(adapter) { const ov::AnyMap& configuration = adapter->getModelConfig(); initDefaultParameters(configuration); } @@ -123,14 +120,18 @@ void ModelYoloX::setStridesGrids() { } } -std::shared_ptr ModelYoloX::preprocess(const InputData& inputData, - InferenceInput& input) { +std::shared_ptr ModelYoloX::preprocess(const InputData& inputData, InferenceInput& input) { const auto& origImg = inputData.asRef().inputImage; - float scale = std::min(static_cast(netInputWidth) / origImg.cols, - static_cast(netInputHeight) / origImg.rows); + float scale = + std::min(static_cast(netInputWidth) / origImg.cols, static_cast(netInputHeight) / origImg.rows); - cv::Mat resizedImage = resizeImageExt(origImg, netInputWidth, netInputHeight, resizeMode, - interpolationMode, nullptr, cv::Scalar(114, 114, 114)); + cv::Mat resizedImage = resizeImageExt(origImg, + netInputWidth, + netInputHeight, + resizeMode, + interpolationMode, + nullptr, + cv::Scalar(114, 114, 114)); input.emplace(inputNames[0], wrapMat2Tensor(resizedImage)); return std::make_shared(origImg.cols, origImg.rows, scale, scale); @@ -183,21 +184,27 @@ std::unique_ptr ModelYoloX::postprocess(InferenceResult& infResult) // Add successful boxes scores.push_back(score); classes.push_back(mainClass); - Anchor trueBox = {outputPtr[startPos + 0] - outputPtr[startPos + 2] / 2, outputPtr[startPos + 1] - outputPtr[startPos + 3] / 2, - outputPtr[startPos + 0] + outputPtr[startPos + 2] / 2, outputPtr[startPos + 1] + outputPtr[startPos + 3] / 2}; - validBoxes.push_back(Anchor({trueBox.left / scale.scaleX, trueBox.top / scale.scaleY, - trueBox.right / scale.scaleX, trueBox.bottom / scale.scaleY})); + Anchor trueBox = {outputPtr[startPos + 0] - outputPtr[startPos + 2] / 2, + outputPtr[startPos + 1] - outputPtr[startPos + 3] / 2, + outputPtr[startPos + 0] + outputPtr[startPos + 2] / 2, + outputPtr[startPos + 1] + outputPtr[startPos + 3] / 2}; + validBoxes.push_back(Anchor({trueBox.left / scale.scaleX, + trueBox.top / scale.scaleY, + trueBox.right / scale.scaleX, + trueBox.bottom / scale.scaleY})); } // NMS for valid boxes const std::vector& keep = nms(validBoxes, scores, iou_threshold, true); - for (size_t index: keep) { + for (size_t index : keep) { // Create new detected box DetectedObject obj; obj.x = clamp(validBoxes[index].left, 0.f, static_cast(scale.inputImgWidth)); obj.y = clamp(validBoxes[index].top, 0.f, static_cast(scale.inputImgHeight)); - obj.height = clamp(validBoxes[index].bottom - validBoxes[index].top, 0.f, static_cast(scale.inputImgHeight)); - obj.width = clamp(validBoxes[index].right - validBoxes[index].left, 0.f, static_cast(scale.inputImgWidth)); + obj.height = + clamp(validBoxes[index].bottom - validBoxes[index].top, 0.f, static_cast(scale.inputImgHeight)); + obj.width = + clamp(validBoxes[index].right - validBoxes[index].left, 0.f, static_cast(scale.inputImgWidth)); obj.confidence = scores[index]; obj.labelID = classes[index]; obj.label = getLabelName(classes[index]); diff --git a/model_api/cpp/models/src/image_model.cpp b/model_api/cpp/models/src/image_model.cpp index fe4fed1b..51997223 100644 --- a/model_api/cpp/models/src/image_model.cpp +++ b/model_api/cpp/models/src/image_model.cpp @@ -16,20 +16,19 @@ #include "models/image_model.h" -#include -#include -#include +#include +#include +#include #include #include - -#include +#include #include -#include +#include -#include "models/results.h" #include "models/input_data.h" #include "models/internal_model_data.h" +#include "models/results.h" ImageModel::ImageModel(const std::string& modelFile, const std::string& resize_type, @@ -42,7 +41,7 @@ ImageModel::ImageModel(const std::string& modelFile, RESIZE_MODE ImageModel::selectResizeMode(const std::string& resize_type) { RESIZE_MODE resize = RESIZE_FILL; if ("crop" == resize_type) { - resize = RESIZE_CROP; + resize = RESIZE_CROP; } else if ("standard" == resize_type) { resize = RESIZE_FILL; } else if ("fit_to_window" == resize_type) { @@ -73,14 +72,16 @@ void ImageModel::init_from_config(const ov::AnyMap& top_priority, const ov::AnyM throw std::runtime_error("pad_value must be in range [0, 255]"); } pad_value = static_cast(pad_value_int); - reverse_input_channels = get_from_any_maps("reverse_input_channels", top_priority, mid_priority, reverse_input_channels); + reverse_input_channels = + get_from_any_maps("reverse_input_channels", top_priority, mid_priority, reverse_input_channels); scale_values = get_from_any_maps("scale_values", top_priority, mid_priority, scale_values); mean_values = get_from_any_maps("mean_values", top_priority, mid_priority, mean_values); } ImageModel::ImageModel(std::shared_ptr& model, const ov::AnyMap& configuration) : ModelBase(model, configuration) { - init_from_config(configuration, model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); + init_from_config(configuration, + model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); } ImageModel::ImageModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) @@ -89,7 +90,8 @@ ImageModel::ImageModel(std::shared_ptr& adapter, const ov::Any } std::unique_ptr ImageModel::inferImage(const ImageInputData& inputData) { - return ModelBase::infer(static_cast(inputData));; + return ModelBase::infer(static_cast(inputData)); + ; } std::vector> ImageModel::inferBatchImage(const std::vector& inputImgs) { @@ -121,16 +123,16 @@ void ImageModel::updateModelInfo() { } std::shared_ptr ImageModel::embedProcessing(std::shared_ptr& model, - const std::string& inputName, - const ov::Layout& layout, - const RESIZE_MODE resize_mode, - const cv::InterpolationFlags interpolationMode, - const ov::Shape& targetShape, - uint8_t pad_value, - bool brg2rgb, - const std::vector& mean, - const std::vector& scale, - const std::type_info& dtype) { + const std::string& inputName, + const ov::Layout& layout, + const RESIZE_MODE resize_mode, + const cv::InterpolationFlags interpolationMode, + const ov::Shape& targetShape, + uint8_t pad_value, + bool brg2rgb, + const std::vector& mean, + const std::vector& scale, + const std::type_info& dtype) { ov::preprocess::PrePostProcessor ppp(model); // Change the input type to the 8-bit image @@ -138,9 +140,7 @@ std::shared_ptr ImageModel::embedProcessing(std::shared_ptr ImageModel::preprocess(const InputData& input if (!useAutoResize && !embedded_processing) { // Resize and copy data from the image to the input tensor - auto tensorShape = inferenceAdapter->getInputShape(inputNames[0]).get_max_shape(); // first input should be image + auto tensorShape = + inferenceAdapter->getInputShape(inputNames[0]).get_max_shape(); // first input should be image const ov::Layout layout("NHWC"); const size_t width = tensorShape[ov::layout::width_idx(layout)]; const size_t height = tensorShape[ov::layout::height_idx(layout)]; const size_t channels = tensorShape[ov::layout::channels_idx(layout)]; if (static_cast(img.channels()) != channels) { - throw std::runtime_error("The number of channels for model input: " + - std::to_string(channels) + " and image: " + - std::to_string(img.channels()) + " - must match"); + throw std::runtime_error("The number of channels for model input: " + std::to_string(channels) + + " and image: " + std::to_string(img.channels()) + " - must match"); } if (channels != 1 && channels != 3) { throw std::runtime_error("Unsupported number of channels"); diff --git a/model_api/cpp/models/src/instance_segmentation.cpp b/model_api/cpp/models/src/instance_segmentation.cpp index c8823280..fe553a13 100644 --- a/model_api/cpp/models/src/instance_segmentation.cpp +++ b/model_api/cpp/models/src/instance_segmentation.cpp @@ -20,17 +20,16 @@ #include #include -#include -#include -#include #include - #include #include #include +#include +#include +#include -#include "models/internal_model_data.h" #include "models/input_data.h" +#include "models/internal_model_data.h" #include "models/results.h" #include "utils/common.hpp" @@ -49,10 +48,10 @@ void append_xai_names(const std::vector>& outputs, std::vec } cv::Rect expand_box(const cv::Rect2f& box, float scale) { - float w_half = box.width * 0.5f * scale, - h_half = box.height * 0.5f * scale; + float w_half = box.width * 0.5f * scale, h_half = box.height * 0.5f * scale; const cv::Point2f& center = (box.tl() + box.br()) * 0.5f; - return {cv::Point(int(center.x - w_half), int(center.y - h_half)), cv::Point(int(center.x + w_half), int(center.y + h_half))}; + return {cv::Point(int(center.x - w_half), int(center.y - h_half)), + cv::Point(int(center.x + w_half), int(center.y + h_half))}; } std::vector> average_and_normalize(const std::vector>& saliency_maps) { @@ -66,9 +65,11 @@ std::vector> average_and_normalize(const std::vector& infResult) { if (pair.first == saliency_map_name || pair.first == feature_vector_name) { continue; } - switch(pair.second.get_shape().size()) { - case 2: - lbm.labels = pair.second; - break; - case 3: - lbm.boxes = pair.second; - break; - case 4: - lbm.masks = pair.second; - break; - case 0: - break; - default: - throw std::runtime_error("Unexpected result: " + pair.first); + switch (pair.second.get_shape().size()) { + case 2: + lbm.labels = pair.second; + break; + case 3: + lbm.boxes = pair.second; + break; + case 4: + lbm.masks = pair.second; + break; + case 0: + break; + default: + throw std::runtime_error("Unexpected result: " + pair.first); } } return lbm; } -} +} // namespace cv::Mat segm_postprocess(const SegmentedObject& box, const cv::Mat& unpadded, int im_h, int im_w) { // Add zero border to prevent upsampling artifacts on segment borders. @@ -137,7 +138,10 @@ cv::Mat segm_postprocess(const SegmentedObject& box, const cv::Mat& unpadded, in cv::Mat resized; cv::resize(raw_cls_mask, resized, {w, h}); cv::Mat im_mask(cv::Size{im_w, im_h}, CV_8UC1, cv::Scalar{0}); - im_mask(cv::Rect{x0, y0, x1-x0, y1-y0}).setTo(1, resized({cv::Point(x0-extended_box.x, y0-extended_box.y), cv::Point(x1-extended_box.x, y1-extended_box.y)}) > 0.5f); + im_mask(cv::Rect{x0, y0, x1 - x0, y1 - y0}) + .setTo(1, + resized({cv::Point(x0 - extended_box.x, y0 - extended_box.y), + cv::Point(x1 - extended_box.x, y1 - extended_box.y)}) > 0.5f); return im_mask; } @@ -145,35 +149,42 @@ std::string MaskRCNNModel::ModelType = "MaskRCNN"; void MaskRCNNModel::init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority) { confidence_threshold = get_from_any_maps("confidence_threshold", top_priority, mid_priority, confidence_threshold); - postprocess_semantic_masks = get_from_any_maps("postprocess_semantic_masks", top_priority, mid_priority, postprocess_semantic_masks); + postprocess_semantic_masks = + get_from_any_maps("postprocess_semantic_masks", top_priority, mid_priority, postprocess_semantic_masks); } MaskRCNNModel::MaskRCNNModel(std::shared_ptr& model, const ov::AnyMap& configuration) - : ImageModel(model, configuration) { - init_from_config(configuration, model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); + : ImageModel(model, configuration) { + init_from_config(configuration, + model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); } MaskRCNNModel::MaskRCNNModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : ImageModel(adapter, configuration) { + : ImageModel(adapter, configuration) { init_from_config(configuration, adapter->getModelConfig()); } -std::unique_ptr MaskRCNNModel::create_model(const std::string& modelFile, const ov::AnyMap& configuration, bool preload, const std::string& device) { +std::unique_ptr MaskRCNNModel::create_model(const std::string& modelFile, + const ov::AnyMap& configuration, + bool preload, + const std::string& device) { auto core = ov::Core(); std::shared_ptr model = core.read_model(modelFile); // Check model_type in the rt_info, ignore configuration std::string model_type = MaskRCNNModel::ModelType; try { - if (model->has_rt_info("model_info", "model_type") ) { + if (model->has_rt_info("model_info", "model_type")) { model_type = model->get_rt_info("model_info", "model_type"); } } catch (const std::exception&) { - slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type << slog::endl; + slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type + << slog::endl; } if (model_type != MaskRCNNModel::ModelType) { - throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + model_type); + throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + + model_type); } std::unique_ptr segmentor{new MaskRCNNModel(model, configuration)}; @@ -224,21 +235,21 @@ void MaskRCNNModel::prepareInputsOutputs(std::shared_ptr& model) { } if (!embedded_processing) { - model = ImageModel::embedProcessing(model, - inputNames[0], - inputLayout, - resizeMode, - interpolationMode, - ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], - inputShape[ov::layout::height_idx(inputLayout)]}, - pad_value, - reverse_input_channels, - mean_values, - scale_values); + model = ImageModel::embedProcessing( + model, + inputNames[0], + inputLayout, + resizeMode, + interpolationMode, + ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], inputShape[ov::layout::height_idx(inputLayout)]}, + pad_value, + reverse_input_channels, + mean_values, + scale_values); netInputWidth = inputShape[ov::layout::width_idx(inputLayout)]; netInputHeight = inputShape[ov::layout::height_idx(inputLayout)]; - useAutoResize = true; // temporal solution + useAutoResize = true; // temporal solution embedded_processing = true; } @@ -251,29 +262,31 @@ void MaskRCNNModel::prepareInputsOutputs(std::shared_ptr& model) { filtered.reserve(3); for (ov::Output& output : model->outputs()) { const std::unordered_set& out_names = output.get_names(); - if (out_names.find(saliency_map_name) == out_names.end() && out_names.find(feature_vector_name) == out_names.end()) { + if (out_names.find(saliency_map_name) == out_names.end() && + out_names.find(feature_vector_name) == out_names.end()) { filtered.push_back({output.get_any_name(), output.get_partial_shape().get_max_shape().size()}); } } if (filtered.size() != 3 && filtered.size() != 4) { - throw std::logic_error(std::string{"MaskRCNNModel model wrapper supports topologies with "} + saliency_map_name + ", " + feature_vector_name + " and 3 or 4 other outputs"); + throw std::logic_error(std::string{"MaskRCNNModel model wrapper supports topologies with "} + + saliency_map_name + ", " + feature_vector_name + " and 3 or 4 other outputs"); } outputNames.resize(3); for (const NameRank& name_rank : filtered) { switch (name_rank.rank) { - case 2: - outputNames[0] = name_rank.name; - break; - case 3: - outputNames[1] = name_rank.name; - break; - case 4: - outputNames[2] = name_rank.name; - break; - case 0: - break; - default: - throw std::runtime_error("Unexpected output: " + name_rank.name); + case 2: + outputNames[0] = name_rank.name; + break; + case 3: + outputNames[1] = name_rank.name; + break; + case 4: + outputNames[2] = name_rank.name; + break; + case 0: + break; + default: + throw std::runtime_error("Unexpected output: " + name_rank.name); } } append_xai_names(model->outputs(), outputNames); @@ -282,9 +295,8 @@ void MaskRCNNModel::prepareInputsOutputs(std::shared_ptr& model) { std::unique_ptr MaskRCNNModel::postprocess(InferenceResult& infResult) { const auto& internalData = infResult.internalModelData->asRef(); float floatInputImgWidth = float(internalData.inputImgWidth), - floatInputImgHeight = float(internalData.inputImgHeight); - float invertedScaleX = floatInputImgWidth / netInputWidth, - invertedScaleY = floatInputImgHeight / netInputHeight; + floatInputImgHeight = float(internalData.inputImgHeight); + float invertedScaleX = floatInputImgWidth / netInputWidth, invertedScaleY = floatInputImgHeight / netInputHeight; int padLeft = 0, padTop = 0; if (RESIZE_KEEP_ASPECT == resizeMode || RESIZE_KEEP_ASPECT_LETTERBOX == resizeMode) { invertedScaleX = invertedScaleY = std::max(invertedScaleX, invertedScaleY); @@ -302,7 +314,8 @@ std::unique_ptr MaskRCNNModel::postprocess(InferenceResult& infResul InstanceSegmentationResult* result = new InstanceSegmentationResult(infResult.frameId, infResult.metaData); auto retVal = std::unique_ptr(result); std::vector> saliency_maps; - bool has_feature_vector_name = std::find(outputNames.begin(), outputNames.end(), feature_vector_name) != outputNames.end(); + bool has_feature_vector_name = + std::find(outputNames.begin(), outputNames.end(), feature_vector_name) != outputNames.end(); if (has_feature_vector_name) { if (labels.empty()) { throw std::runtime_error("Can't get number of classes because labels are empty"); @@ -323,21 +336,12 @@ std::unique_ptr MaskRCNNModel::postprocess(InferenceResult& infResul } obj.label = getLabelName(obj.labelID); - obj.x = clamp( - round((boxes[i * objectSize + 0] - padLeft) * invertedScaleX), - 0.f, - floatInputImgWidth); - obj.y = clamp( - round((boxes[i * objectSize + 1] - padTop) * invertedScaleY), - 0.f, - floatInputImgHeight); - obj.width = clamp( - round((boxes[i * objectSize + 2] - padLeft) * invertedScaleX - obj.x), - 0.f, - floatInputImgWidth); - obj.height = clamp( - round((boxes[i * objectSize + 3] - padTop) * invertedScaleY - obj.y), - 0.f, floatInputImgHeight); + obj.x = clamp(round((boxes[i * objectSize + 0] - padLeft) * invertedScaleX), 0.f, floatInputImgWidth); + obj.y = clamp(round((boxes[i * objectSize + 1] - padTop) * invertedScaleY), 0.f, floatInputImgHeight); + obj.width = + clamp(round((boxes[i * objectSize + 2] - padLeft) * invertedScaleX - obj.x), 0.f, floatInputImgWidth); + obj.height = + clamp(round((boxes[i * objectSize + 3] - padTop) * invertedScaleY - obj.y), 0.f, floatInputImgHeight); if (obj.height * obj.width <= 1) { continue; @@ -370,7 +374,8 @@ std::unique_ptr MaskRCNNModel::infer(const ImageInpu return std::unique_ptr(static_cast(result.release())); } -std::vector> MaskRCNNModel::inferBatch(const std::vector& inputImgs) { +std::vector> MaskRCNNModel::inferBatch( + const std::vector& inputImgs) { auto results = ImageModel::inferBatchImage(inputImgs); std::vector> isegResults; isegResults.reserve(results.size()); diff --git a/model_api/cpp/models/src/keypoint_detection.cpp b/model_api/cpp/models/src/keypoint_detection.cpp index 8247e2f7..a8daa337 100644 --- a/model_api/cpp/models/src/keypoint_detection.cpp +++ b/model_api/cpp/models/src/keypoint_detection.cpp @@ -16,12 +16,11 @@ #include "models/keypoint_detection.h" -#include -#include - #include #include #include +#include +#include #include "models/input_data.h" #include "models/internal_model_data.h" @@ -35,7 +34,7 @@ void colArgMax(const cv::Mat& src, cv::Mat& dst_locs, cv::Mat& dst_values) { dst_values = cv::Mat::zeros(src.rows, 1, CV_32F); for (int row = 0; row < src.rows; row++) { - const float *ptr_row = src.ptr(row); + const float* ptr_row = src.ptr(row); int max_val_idx = 0; dst_values.at(row) = ptr_row[max_val_idx]; for (int col = 1; col < src.cols; ++col) { @@ -48,7 +47,8 @@ void colArgMax(const cv::Mat& src, cv::Mat& dst_locs, cv::Mat& dst_values) { } } -DetectedKeypoints decode_simcc(const cv::Mat& simcc_x, const cv::Mat& simcc_y, +DetectedKeypoints decode_simcc(const cv::Mat& simcc_x, + const cv::Mat& simcc_y, const cv::Point2f& extra_scale = cv::Point2f(1.f, 1.f), float simcc_split_ratio = 2.0f) { cv::Mat x_locs, max_val_x; @@ -60,7 +60,8 @@ DetectedKeypoints decode_simcc(const cv::Mat& simcc_x, const cv::Mat& simcc_y, std::vector keypoints(x_locs.rows); cv::Mat scores = cv::Mat::zeros(x_locs.rows, 1, CV_32F); for (int i = 0; i < x_locs.rows; i++) { - keypoints[i] = cv::Point2f(x_locs.at(i) * extra_scale.x, y_locs.at(i) * extra_scale.y) / simcc_split_ratio; + keypoints[i] = + cv::Point2f(x_locs.at(i) * extra_scale.x, y_locs.at(i) * extra_scale.y) / simcc_split_ratio; scores.at(i) = std::min(max_val_x.at(i), max_val_y.at(i)); if (scores.at(i) <= 0.f) { @@ -71,7 +72,7 @@ DetectedKeypoints decode_simcc(const cv::Mat& simcc_x, const cv::Mat& simcc_y, return {std::move(keypoints), scores}; } -} +} // namespace std::string KeypointDetectionModel::ModelType = "keypoint_detection"; @@ -79,31 +80,39 @@ void KeypointDetectionModel::init_from_config(const ov::AnyMap& top_priority, co labels = get_from_any_maps("labels", top_priority, mid_priority, labels); } -KeypointDetectionModel::KeypointDetectionModel(std::shared_ptr& model, const ov::AnyMap& configuration) : ImageModel(model, configuration) { - init_from_config(configuration, model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); +KeypointDetectionModel::KeypointDetectionModel(std::shared_ptr& model, const ov::AnyMap& configuration) + : ImageModel(model, configuration) { + init_from_config(configuration, + model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); } -KeypointDetectionModel::KeypointDetectionModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : ImageModel(adapter, configuration) { +KeypointDetectionModel::KeypointDetectionModel(std::shared_ptr& adapter, + const ov::AnyMap& configuration) + : ImageModel(adapter, configuration) { init_from_config(configuration, adapter->getModelConfig()); } -std::unique_ptr KeypointDetectionModel::create_model(const std::string& modelFile, const ov::AnyMap& configuration, bool preload, const std::string& device) { +std::unique_ptr KeypointDetectionModel::create_model(const std::string& modelFile, + const ov::AnyMap& configuration, + bool preload, + const std::string& device) { auto core = ov::Core(); std::shared_ptr model = core.read_model(modelFile); // Check model_type in the rt_info, ignore configuration std::string model_type = KeypointDetectionModel::ModelType; try { - if (model->has_rt_info("model_info", "model_type") ) { + if (model->has_rt_info("model_info", "model_type")) { model_type = model->get_rt_info("model_info", "model_type"); } } catch (const std::exception&) { - slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type << slog::endl; + slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type + << slog::endl; } if (model_type != KeypointDetectionModel::ModelType) { - throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + model_type); + throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + + model_type); } std::unique_ptr kp_detector{new KeypointDetectionModel(model, configuration)}; @@ -114,7 +123,8 @@ std::unique_ptr KeypointDetectionModel::create_model(con return kp_detector; } -std::unique_ptr KeypointDetectionModel::create_model(std::shared_ptr& adapter) { +std::unique_ptr KeypointDetectionModel::create_model( + std::shared_ptr& adapter) { const ov::AnyMap& configuration = adapter->getModelConfig(); auto model_type_iter = configuration.find("model_type"); std::string model_type = KeypointDetectionModel::ModelType; @@ -141,7 +151,8 @@ void KeypointDetectionModel::prepareInputsOutputs(std::shared_ptr& mo // --------------------------- Configure input & output --------------------------------------------- // --------------------------- Prepare input ----------------------------------------------------- if (model->inputs().size() != 1) { - throw std::logic_error(KeypointDetectionModel::ModelType + " model wrapper supports topologies with only 1 input"); + throw std::logic_error(KeypointDetectionModel::ModelType + + " model wrapper supports topologies with only 1 input"); } const auto& input = model->input(); inputNames.push_back(input.get_any_name()); @@ -156,17 +167,17 @@ void KeypointDetectionModel::prepareInputsOutputs(std::shared_ptr& mo } if (!embedded_processing) { - model = ImageModel::embedProcessing(model, - inputNames[0], - inputLayout, - resizeMode, - interpolationMode, - ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], - inputShape[ov::layout::height_idx(inputLayout)]}, - pad_value, - reverse_input_channels, - mean_values, - scale_values); + model = ImageModel::embedProcessing( + model, + inputNames[0], + inputLayout, + resizeMode, + interpolationMode, + ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], inputShape[ov::layout::height_idx(inputLayout)]}, + pad_value, + reverse_input_channels, + mean_values, + scale_values); ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); model = ppp.build(); @@ -188,13 +199,17 @@ std::unique_ptr KeypointDetectionModel::postprocess(InferenceResult& size_t shape_offset = pred_x_tensor.get_shape().size() == 3 ? 1 : 0; auto pred_x_mat = cv::Mat(cv::Size(static_cast(pred_x_tensor.get_shape()[shape_offset + 1]), static_cast(pred_x_tensor.get_shape()[shape_offset])), - CV_32F, pred_x_tensor.data(), pred_x_tensor.get_strides()[shape_offset]); + CV_32F, + pred_x_tensor.data(), + pred_x_tensor.get_strides()[shape_offset]); const ov::Tensor& pred_y_tensor = infResult.outputsData.find(outputNames[1])->second; shape_offset = pred_y_tensor.get_shape().size() == 3 ? 1 : 0; auto pred_y_mat = cv::Mat(cv::Size(static_cast(pred_y_tensor.get_shape()[shape_offset + 1]), static_cast(pred_y_tensor.get_shape()[shape_offset])), - CV_32F, pred_y_tensor.data(), pred_y_tensor.get_strides()[shape_offset]); + CV_32F, + pred_y_tensor.data(), + pred_y_tensor.get_strides()[shape_offset]); const auto& image_data = infResult.internalModelData->asRef(); float inverted_scale_x = static_cast(image_data.inputImgWidth) / netInputWidth, @@ -204,14 +219,13 @@ std::unique_ptr KeypointDetectionModel::postprocess(InferenceResult& return std::unique_ptr(result); } - -std::unique_ptr -KeypointDetectionModel::infer(const ImageInputData& inputData) { +std::unique_ptr KeypointDetectionModel::infer(const ImageInputData& inputData) { auto result = ImageModel::inferImage(inputData); return std::unique_ptr(static_cast(result.release())); } -std::vector> KeypointDetectionModel::inferBatch(const std::vector& inputImgs) { +std::vector> KeypointDetectionModel::inferBatch( + const std::vector& inputImgs) { auto results = ImageModel::inferBatchImage(inputImgs); std::vector> kpDetResults; kpDetResults.reserve(results.size()); diff --git a/model_api/cpp/models/src/model_base.cpp b/model_api/cpp/models/src/model_base.cpp index 2af5c9d7..7fead2ef 100644 --- a/model_api/cpp/models/src/model_base.cpp +++ b/model_api/cpp/models/src/model_base.cpp @@ -15,51 +15,50 @@ */ #include "models/model_base.h" -#include -#include "utils/args_helper.hpp" -#include "models/input_data.h" -#include -#include +#include +#include #include - +#include #include #include #include +#include "models/input_data.h" +#include "utils/args_helper.hpp" namespace { class TmpCallbackSetter { - public: - ModelBase* model; - std::function, const ov::AnyMap&)> last_callback; +public: + ModelBase* model; + std::function, const ov::AnyMap&)> last_callback; TmpCallbackSetter(ModelBase* model_, std::function, const ov::AnyMap&)> tmp_callback, std::function, const ov::AnyMap&)> last_callback_) - : model(model_), last_callback(last_callback_) { + : model(model_), + last_callback(last_callback_) { model->setCallback(tmp_callback); } ~TmpCallbackSetter() { if (last_callback) { model->setCallback(last_callback); - } - else { - model->setCallback([](std::unique_ptr, const ov::AnyMap&){}); + } else { + model->setCallback([](std::unique_ptr, const ov::AnyMap&) {}); } } }; -} +} // namespace ModelBase::ModelBase(const std::string& modelFile, const std::string& layout) - : modelFile(modelFile), - inputsLayouts(parseLayoutString(layout)) { + : modelFile(modelFile), + inputsLayouts(parseLayoutString(layout)) { auto core = ov::Core(); model = core.read_model(modelFile); } ModelBase::ModelBase(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : inferenceAdapter(adapter) { + : inferenceAdapter(adapter) { const ov::AnyMap& adapter_configuration = adapter->getModelConfig(); std::string layout = ""; @@ -70,13 +69,12 @@ ModelBase::ModelBase(std::shared_ptr& adapter, const ov::AnyMa outputNames = adapter->getOutputNames(); } -ModelBase::ModelBase(std::shared_ptr& model, const ov::AnyMap& configuration) - : model(model) { +ModelBase::ModelBase(std::shared_ptr& model, const ov::AnyMap& configuration) : model(model) { auto layout_iter = configuration.find("layout"); std::string layout = ""; if (layout_iter != configuration.end()) { - layout = layout_iter->second.as(); + layout = layout_iter->second.as(); } else { if (model->has_rt_info("model_info", "layout")) { layout = model->get_rt_info("model_info", "layout"); @@ -145,10 +143,12 @@ std::unique_ptr ModelBase::infer(const InputData& inputData) { return retVal; } -std::vector> ModelBase::inferBatch(const std::vector>& inputData) { +std::vector> ModelBase::inferBatch( + const std::vector>& inputData) { auto results = std::vector>(inputData.size()); - auto setter = TmpCallbackSetter(this, - [&](std::unique_ptr result, const ov::AnyMap& callback_args){ + auto setter = TmpCallbackSetter( + this, + [&](std::unique_ptr result, const ov::AnyMap& callback_args) { size_t id = callback_args.find("id")->second.as(); results[id] = std::move(result); }, @@ -170,7 +170,6 @@ std::vector> ModelBase::inferBatch(const std::vector return inferBatch(inputRefData); } - void ModelBase::inferAsync(const InputData& inputData, const ov::AnyMap& callback_args) { InferenceInput inputs; auto internalModelData = this->preprocess(inputData, inputs); @@ -188,7 +187,8 @@ void ModelBase::awaitAll() { void ModelBase::awaitAny() { inferenceAdapter->awaitAny(); } -void ModelBase::setCallback(std::function, const ov::AnyMap& callback_args)> callback) { +void ModelBase::setCallback( + std::function, const ov::AnyMap& callback_args)> callback) { lastCallback = callback; inferenceAdapter->setCallback([this, callback](ov::InferRequest request, CallbackData args) { InferenceResult result; @@ -215,7 +215,8 @@ size_t ModelBase::getNumAsyncExecutors() const { std::shared_ptr ModelBase::getModel() { if (!model) { - throw std::runtime_error(std::string("ov::Model is not accessible for the current model adapter: ") + typeid(inferenceAdapter).name()); + throw std::runtime_error(std::string("ov::Model is not accessible for the current model adapter: ") + + typeid(inferenceAdapter).name()); } updateModelInfo(); diff --git a/model_api/cpp/models/src/segmentation_model.cpp b/model_api/cpp/models/src/segmentation_model.cpp index 0d7bdbbb..0687e252 100644 --- a/model_api/cpp/models/src/segmentation_model.cpp +++ b/model_api/cpp/models/src/segmentation_model.cpp @@ -20,13 +20,12 @@ #include #include -#include -#include -#include - #include #include #include +#include +#include +#include #include "models/input_data.h" #include "models/internal_model_data.h" @@ -45,9 +44,11 @@ cv::Mat get_activation_map(const cv::Mat& features) { features.convertTo(int_act_map, CV_8U, factor, -min_soft_score * factor); return int_act_map; } -} +} // namespace -cv::Mat create_hard_prediction_from_soft_prediction(const cv::Mat& soft_prediction, float soft_threshold, int blur_strength) { +cv::Mat create_hard_prediction_from_soft_prediction(const cv::Mat& soft_prediction, + float soft_threshold, + int blur_strength) { if (soft_prediction.channels() == 1) { return soft_prediction; } @@ -86,34 +87,42 @@ std::string SegmentationModel::ModelType = "Segmentation"; void SegmentationModel::init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority) { blur_strength = get_from_any_maps("blur_strength", top_priority, mid_priority, blur_strength); soft_threshold = get_from_any_maps("soft_threshold", top_priority, mid_priority, soft_threshold); - return_soft_prediction = get_from_any_maps("return_soft_prediction", top_priority, mid_priority, return_soft_prediction); + return_soft_prediction = + get_from_any_maps("return_soft_prediction", top_priority, mid_priority, return_soft_prediction); } -SegmentationModel::SegmentationModel(std::shared_ptr& model, const ov::AnyMap& configuration) : ImageModel(model, configuration) { - init_from_config(configuration, model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); +SegmentationModel::SegmentationModel(std::shared_ptr& model, const ov::AnyMap& configuration) + : ImageModel(model, configuration) { + init_from_config(configuration, + model->has_rt_info("model_info") ? model->get_rt_info("model_info") : ov::AnyMap{}); } SegmentationModel::SegmentationModel(std::shared_ptr& adapter, const ov::AnyMap& configuration) - : ImageModel(adapter, configuration) { + : ImageModel(adapter, configuration) { init_from_config(configuration, adapter->getModelConfig()); } -std::unique_ptr SegmentationModel::create_model(const std::string& modelFile, const ov::AnyMap& configuration, bool preload, const std::string& device) { +std::unique_ptr SegmentationModel::create_model(const std::string& modelFile, + const ov::AnyMap& configuration, + bool preload, + const std::string& device) { auto core = ov::Core(); std::shared_ptr model = core.read_model(modelFile); // Check model_type in the rt_info, ignore configuration std::string model_type = SegmentationModel::ModelType; try { - if (model->has_rt_info("model_info", "model_type") ) { + if (model->has_rt_info("model_info", "model_type")) { model_type = model->get_rt_info("model_info", "model_type"); } } catch (const std::exception&) { - slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type << slog::endl; + slog::warn << "Model type is not specified in the rt_info, use default model type: " << model_type + << slog::endl; } if (model_type != SegmentationModel::ModelType) { - throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + model_type); + throw std::runtime_error("Incorrect or unsupported model_type is provided in the model_info section: " + + model_type); } std::unique_ptr segmentor{new SegmentationModel(model, configuration)}; @@ -174,7 +183,8 @@ void SegmentationModel::prepareInputsOutputs(std::shared_ptr& model) if (out_name.empty()) { out_name = output.get_any_name(); } else { - throw std::runtime_error(std::string{"Only "} + feature_vector_name + " and 1 other output are allowed"); + throw std::runtime_error(std::string{"Only "} + feature_vector_name + + " and 1 other output are allowed"); } } } @@ -183,17 +193,17 @@ void SegmentationModel::prepareInputsOutputs(std::shared_ptr& model) } if (!embedded_processing) { - model = ImageModel::embedProcessing(model, - inputNames[0], - inputLayout, - resizeMode, - interpolationMode, - ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], - inputShape[ov::layout::height_idx(inputLayout)]}, - pad_value, - reverse_input_channels, - mean_values, - scale_values); + model = ImageModel::embedProcessing( + model, + inputNames[0], + inputLayout, + resizeMode, + interpolationMode, + ov::Shape{inputShape[ov::layout::width_idx(inputLayout)], inputShape[ov::layout::height_idx(inputLayout)]}, + pad_value, + reverse_input_channels, + mean_values, + scale_values); ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); ov::Layout out_layout = getLayoutFromShape(model->output(out_name).get_partial_shape()); @@ -206,7 +216,7 @@ void SegmentationModel::prepareInputsOutputs(std::shared_ptr& model) ppp.output(out_name).tensor().set_layout("NHW"); } model = ppp.build(); - useAutoResize = true; // temporal solution + useAutoResize = true; // temporal solution embedded_processing = true; } @@ -226,8 +236,8 @@ std::unique_ptr SegmentationModel::postprocess(InferenceResult& infR const auto& outTensor = infResult.outputsData[outputName]; const ov::Shape& outputShape = outTensor.get_shape(); const ov::Layout& outputLayout = getLayoutFromShape(outputShape); - size_t outChannels = ov::layout::has_channels(outputLayout) ? - outputShape[ov::layout::channels_idx(outputLayout)] : 1; + size_t outChannels = + ov::layout::has_channels(outputLayout) ? outputShape[ov::layout::channels_idx(outputLayout)] : 1; int outHeight = static_cast(outputShape[ov::layout::height_idx(outputLayout)]); int outWidth = static_cast(outputShape[ov::layout::width_idx(outputLayout)]); cv::Mat soft_prediction; @@ -250,14 +260,26 @@ std::unique_ptr SegmentationModel::postprocess(InferenceResult& infR cv::merge(channels, soft_prediction); } - cv::Mat hard_prediction = create_hard_prediction_from_soft_prediction(soft_prediction, soft_threshold, blur_strength); + cv::Mat hard_prediction = + create_hard_prediction_from_soft_prediction(soft_prediction, soft_threshold, blur_strength); - cv::resize(hard_prediction, hard_prediction, {inputImgSize.inputImgWidth, inputImgSize.inputImgHeight}, 0.0, 0.0, cv::INTER_NEAREST); + cv::resize(hard_prediction, + hard_prediction, + {inputImgSize.inputImgWidth, inputImgSize.inputImgHeight}, + 0.0, + 0.0, + cv::INTER_NEAREST); if (return_soft_prediction) { - ImageResultWithSoftPrediction* result = new ImageResultWithSoftPrediction(infResult.frameId, infResult.metaData); + ImageResultWithSoftPrediction* result = + new ImageResultWithSoftPrediction(infResult.frameId, infResult.metaData); result->resultImage = hard_prediction; - cv::resize(soft_prediction, soft_prediction, {inputImgSize.inputImgWidth, inputImgSize.inputImgHeight}, 0.0, 0.0, cv::INTER_NEAREST); + cv::resize(soft_prediction, + soft_prediction, + {inputImgSize.inputImgWidth, inputImgSize.inputImgHeight}, + 0.0, + 0.0, + cv::INTER_NEAREST); result->soft_prediction = soft_prediction; auto iter = infResult.outputsData.find(feature_vector_name); if (infResult.outputsData.end() != iter) { @@ -272,7 +294,7 @@ std::unique_ptr SegmentationModel::postprocess(InferenceResult& infR return std::unique_ptr(result); } -std::vector SegmentationModel::getContours(const ImageResultWithSoftPrediction &imageResult) { +std::vector SegmentationModel::getContours(const ImageResultWithSoftPrediction& imageResult) { if (imageResult.soft_prediction.channels() == 1) { throw std::runtime_error{"Cannot get contours from soft prediction with 1 layer"}; } @@ -282,26 +304,29 @@ std::vector SegmentationModel::getContours(const ImageResultWithSoftPre cv::Mat current_label_soft_prediction; for (int index = 1; index < imageResult.soft_prediction.channels(); index++) { cv::extractChannel(imageResult.soft_prediction, current_label_soft_prediction, index); - cv::inRange(imageResult.resultImage, cv::Scalar(index, index, index), cv::Scalar(index, index, index), label_index_map); + cv::inRange(imageResult.resultImage, + cv::Scalar(index, index, index), + cv::Scalar(index, index, index), + label_index_map); std::vector> contours; cv::findContours(label_index_map, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_NONE); std::string label = getLabelName(index - 1); for (unsigned int i = 0; i < contours.size(); i++) { - cv::Mat mask = cv::Mat::zeros(imageResult.resultImage.rows, imageResult.resultImage.cols, imageResult.resultImage.type()); + cv::Mat mask = cv::Mat::zeros(imageResult.resultImage.rows, + imageResult.resultImage.cols, + imageResult.resultImage.type()); cv::drawContours(mask, contours, i, 255, -1); float probability = (float)cv::mean(current_label_soft_prediction, mask)[0]; combined_contours.push_back({label, probability, contours[i]}); } - } return combined_contours; } -std::unique_ptr -SegmentationModel::infer(const ImageInputData& inputData) { +std::unique_ptr SegmentationModel::infer(const ImageInputData& inputData) { auto result = ImageModel::inferImage(inputData); return std::unique_ptr(static_cast(result.release())); } diff --git a/model_api/cpp/tilers/include/tilers/detection.h b/model_api/cpp/tilers/include/tilers/detection.h index eed09086..7799961a 100644 --- a/model_api/cpp/tilers/include/tilers/detection.h +++ b/model_api/cpp/tilers/include/tilers/detection.h @@ -28,8 +28,12 @@ class DetectionTiler : public TilerBase { protected: virtual std::unique_ptr postprocess_tile(std::unique_ptr, const cv::Rect&); - virtual std::unique_ptr merge_results(const std::vector>&, const cv::Size&, const std::vector&); - ov::Tensor merge_saliency_maps(const std::vector>&, const cv::Size&, const std::vector&); + virtual std::unique_ptr merge_results(const std::vector>&, + const cv::Size&, + const std::vector&); + ov::Tensor merge_saliency_maps(const std::vector>&, + const cv::Size&, + const std::vector&); size_t max_pred_number = 200; }; diff --git a/model_api/cpp/tilers/include/tilers/instance_segmentation.h b/model_api/cpp/tilers/include/tilers/instance_segmentation.h index 3b5f5878..c87fa03e 100644 --- a/model_api/cpp/tilers/include/tilers/instance_segmentation.h +++ b/model_api/cpp/tilers/include/tilers/instance_segmentation.h @@ -29,9 +29,13 @@ class InstanceSegmentationTiler : public TilerBase { protected: virtual std::unique_ptr postprocess_tile(std::unique_ptr, const cv::Rect&); - virtual std::unique_ptr merge_results(const std::vector>&, const cv::Size&, const std::vector&); + virtual std::unique_ptr merge_results(const std::vector>&, + const cv::Size&, + const std::vector&); - std::vector> merge_saliency_maps(const std::vector>&, const cv::Size&, const std::vector&); + std::vector> merge_saliency_maps(const std::vector>&, + const cv::Size&, + const std::vector&); size_t max_pred_number = 200; }; diff --git a/model_api/cpp/tilers/include/tilers/semantic_segmentation.h b/model_api/cpp/tilers/include/tilers/semantic_segmentation.h index a3cecd47..74701f87 100644 --- a/model_api/cpp/tilers/include/tilers/semantic_segmentation.h +++ b/model_api/cpp/tilers/include/tilers/semantic_segmentation.h @@ -28,7 +28,9 @@ class SemanticSegmentationTiler : public TilerBase { protected: virtual std::unique_ptr postprocess_tile(std::unique_ptr, const cv::Rect&); - virtual std::unique_ptr merge_results(const std::vector>&, const cv::Size&, const std::vector&); + virtual std::unique_ptr merge_results(const std::vector>&, + const cv::Size&, + const std::vector&); int blur_strength = -1; float soft_threshold = -std::numeric_limits::infinity(); diff --git a/model_api/cpp/tilers/include/tilers/tiler_base.h b/model_api/cpp/tilers/include/tilers/tiler_base.h index 9af9f6c1..976fd3b4 100644 --- a/model_api/cpp/tilers/include/tilers/tiler_base.h +++ b/model_api/cpp/tilers/include/tilers/tiler_base.h @@ -15,16 +15,15 @@ */ #pragma once +#include + #include #include -#include -#include - #include - +#include #include #include -#include +#include struct ResultBase; @@ -32,14 +31,13 @@ enum class ExecutionMode { sync, async }; class TilerBase { public: - TilerBase(const std::shared_ptr& model, const ov::AnyMap& configuration, + TilerBase(const std::shared_ptr& model, + const ov::AnyMap& configuration, ExecutionMode exec_mode = ExecutionMode::async); virtual ~TilerBase() = default; - protected: - virtual std::unique_ptr run_impl(const ImageInputData& inputData); std::vector tile(const cv::Size&); std::vector filter_tiles(const cv::Mat&, const std::vector&); @@ -47,7 +45,9 @@ class TilerBase { std::unique_ptr predict_async(const cv::Mat&, const std::vector&); cv::Mat crop_tile(const cv::Mat&, const cv::Rect&); virtual std::unique_ptr postprocess_tile(std::unique_ptr, const cv::Rect&) = 0; - virtual std::unique_ptr merge_results(const std::vector>&, const cv::Size&, const std::vector&) = 0; + virtual std::unique_ptr merge_results(const std::vector>&, + const cv::Size&, + const std::vector&) = 0; std::shared_ptr model; size_t tile_size = 400; diff --git a/model_api/cpp/tilers/src/detection.cpp b/model_api/cpp/tilers/src/detection.cpp index 9faeee52..2e53556c 100644 --- a/model_api/cpp/tilers/src/detection.cpp +++ b/model_api/cpp/tilers/src/detection.cpp @@ -14,15 +14,14 @@ // limitations under the License. */ +#include +#include + #include #include -#include #include - -#include -#include #include - +#include namespace { @@ -37,24 +36,23 @@ cv::Mat non_linear_normalization(cv::Mat& class_map) { return class_map; } -} - -DetectionTiler::DetectionTiler(const std::shared_ptr& _model, const ov::AnyMap& configuration) : - TilerBase(_model, configuration) { +} // namespace +DetectionTiler::DetectionTiler(const std::shared_ptr& _model, const ov::AnyMap& configuration) + : TilerBase(_model, configuration) { ov::AnyMap extra_config; try { auto ov_model = model->getModel(); extra_config = ov_model->get_rt_info("model_info"); - } - catch (const std::runtime_error&) { + } catch (const std::runtime_error&) { extra_config = model->getInferenceAdapter()->getModelConfig(); } max_pred_number = get_from_any_maps("max_pred_number", configuration, extra_config, max_pred_number); } -std::unique_ptr DetectionTiler::postprocess_tile(std::unique_ptr tile_result, const cv::Rect& coord) { +std::unique_ptr DetectionTiler::postprocess_tile(std::unique_ptr tile_result, + const cv::Rect& coord) { DetectionResult* det_res = static_cast(tile_result.get()); for (auto& det : det_res->objects) { det.x += coord.x; @@ -62,7 +60,8 @@ std::unique_ptr DetectionTiler::postprocess_tile(std::unique_ptrfeature_vector) { - auto tmp_feature_vector = ov::Tensor(det_res->feature_vector.get_element_type(), det_res->feature_vector.get_shape()); + auto tmp_feature_vector = + ov::Tensor(det_res->feature_vector.get_element_type(), det_res->feature_vector.get_shape()); det_res->feature_vector.copy_to(tmp_feature_vector); det_res->feature_vector = tmp_feature_vector; } @@ -76,7 +75,9 @@ std::unique_ptr DetectionTiler::postprocess_tile(std::unique_ptr DetectionTiler::merge_results(const std::vector>& tiles_results, const cv::Size& image_size, const std::vector& tile_coords) { +std::unique_ptr DetectionTiler::merge_results(const std::vector>& tiles_results, + const cv::Size& image_size, + const std::vector& tile_coords) { DetectionResult* result = new DetectionResult(); auto retVal = std::unique_ptr(result); @@ -103,7 +104,8 @@ std::unique_ptr DetectionTiler::merge_results(const std::vector(tiles_results.begin()->get()); if (det_res->feature_vector) { - result->feature_vector = ov::Tensor(det_res->feature_vector.get_element_type(), det_res->feature_vector.get_shape()); + result->feature_vector = + ov::Tensor(det_res->feature_vector.get_element_type(), det_res->feature_vector.get_shape()); } if (det_res->saliency_map) { result->saliency_map = merge_saliency_maps(tiles_results, image_size, tile_coords); @@ -133,7 +135,9 @@ std::unique_ptr DetectionTiler::merge_results(const std::vector>& tiles_results, const cv::Size& image_size, const std::vector& tile_coords) { +ov::Tensor DetectionTiler::merge_saliency_maps(const std::vector>& tiles_results, + const cv::Size& image_size, + const std::vector& tile_coords) { std::vector all_saliency_maps; all_saliency_maps.reserve(tiles_results.size()); for (const auto& result : tiles_results) { @@ -173,13 +177,19 @@ ov::Tensor DetectionTiler::merge_saliency_maps(const std::vector(tile_coords[i].x * ratio_w), - static_cast(tile_coords[i].y * ratio_h), - static_cast(static_cast(tile_coords[i].width + tile_coords[i].x) * ratio_w - static_cast(tile_coords[i].x * ratio_w)), - static_cast(static_cast(tile_coords[i].height + tile_coords[i].y) * ratio_h - static_cast(tile_coords[i].y * ratio_h))); - - if (current_cls_map_mat.rows > map_location.height && map_location.height > 0 && current_cls_map_mat.cols > map_location.width && map_location.width > 0) { - cv::resize(current_cls_map_mat_float, current_cls_map_mat_float, cv::Size(map_location.width, map_location.height)); + cv::Rect map_location( + static_cast(tile_coords[i].x * ratio_w), + static_cast(tile_coords[i].y * ratio_h), + static_cast(static_cast(tile_coords[i].width + tile_coords[i].x) * ratio_w - + static_cast(tile_coords[i].x * ratio_w)), + static_cast(static_cast(tile_coords[i].height + tile_coords[i].y) * ratio_h - + static_cast(tile_coords[i].y * ratio_h))); + + if (current_cls_map_mat.rows > map_location.height && map_location.height > 0 && + current_cls_map_mat.cols > map_location.width && map_location.width > 0) { + cv::resize(current_cls_map_mat_float, + current_cls_map_mat_float, + cv::Size(map_location.width, map_location.height)); } auto class_map_roi = cv::Mat(merged_map_mat[class_idx], map_location); @@ -187,9 +197,9 @@ ov::Tensor DetectionTiler::merge_saliency_maps(const std::vector(row_i, col_i); if (merged_mixel > 0) { - class_map_roi.at(row_i, col_i) = 0.5f * (merged_mixel + current_cls_map_mat_float.at(row_i, col_i)); - } - else { + class_map_roi.at(row_i, col_i) = + 0.5f * (merged_mixel + current_cls_map_mat_float.at(row_i, col_i)); + } else { class_map_roi.at(row_i, col_i) = current_cls_map_mat_float.at(row_i, col_i); } } @@ -200,8 +210,7 @@ ov::Tensor DetectionTiler::merge_saliency_maps(const std::vector +#include +#include + #include #include -#include #include - -#include -#include -#include #include +#include + #include "utils/common.hpp" namespace { class MaskRCNNModelParamsSetter { - public: - std::shared_ptr model; - bool state; - MaskRCNNModel* model_ptr; +public: + std::shared_ptr model; + bool state; + MaskRCNNModel* model_ptr; MaskRCNNModelParamsSetter(std::shared_ptr model_) : model(model_) { model_ptr = static_cast(model.get()); state = model_ptr->postprocess_semantic_masks; @@ -40,21 +41,21 @@ class MaskRCNNModelParamsSetter { model_ptr->postprocess_semantic_masks = state; } }; -} +} // namespace -InstanceSegmentationTiler::InstanceSegmentationTiler(std::shared_ptr _model, const ov::AnyMap& configuration) : - TilerBase(_model, configuration) { +InstanceSegmentationTiler::InstanceSegmentationTiler(std::shared_ptr _model, + const ov::AnyMap& configuration) + : TilerBase(_model, configuration) { ov::AnyMap extra_config; try { auto ov_model = model->getModel(); extra_config = ov_model->get_rt_info("model_info"); - } - catch (const std::runtime_error&) { + } catch (const std::runtime_error&) { extra_config = model->getInferenceAdapter()->getModelConfig(); } - postprocess_semantic_masks = get_from_any_maps("postprocess_semantic_masks", - configuration, extra_config, postprocess_semantic_masks); + postprocess_semantic_masks = + get_from_any_maps("postprocess_semantic_masks", configuration, extra_config, postprocess_semantic_masks); max_pred_number = get_from_any_maps("max_pred_number", configuration, extra_config, max_pred_number); } @@ -64,7 +65,8 @@ std::unique_ptr InstanceSegmentationTiler::run(const return std::unique_ptr(static_cast(result.release())); } -std::unique_ptr InstanceSegmentationTiler::postprocess_tile(std::unique_ptr tile_result, const cv::Rect& coord) { +std::unique_ptr InstanceSegmentationTiler::postprocess_tile(std::unique_ptr tile_result, + const cv::Rect& coord) { auto* iseg_res = static_cast(tile_result.get()); for (auto& det : iseg_res->segmentedObjects) { det.x += coord.x; @@ -72,7 +74,8 @@ std::unique_ptr InstanceSegmentationTiler::postprocess_tile(std::uni } if (iseg_res->feature_vector) { - auto tmp_feature_vector = ov::Tensor(iseg_res->feature_vector.get_element_type(), iseg_res->feature_vector.get_shape()); + auto tmp_feature_vector = + ov::Tensor(iseg_res->feature_vector.get_element_type(), iseg_res->feature_vector.get_shape()); iseg_res->feature_vector.copy_to(tmp_feature_vector); iseg_res->feature_vector = tmp_feature_vector; } @@ -80,8 +83,10 @@ std::unique_ptr InstanceSegmentationTiler::postprocess_tile(std::uni return tile_result; } -std::unique_ptr InstanceSegmentationTiler::merge_results(const std::vector>& tiles_results, - const cv::Size& image_size, const std::vector& tile_coords) { +std::unique_ptr InstanceSegmentationTiler::merge_results( + const std::vector>& tiles_results, + const cv::Size& image_size, + const std::vector& tile_coords) { auto* result = new InstanceSegmentationResult(); auto retVal = std::unique_ptr(result); @@ -103,8 +108,10 @@ std::unique_ptr InstanceSegmentationTiler::merge_results(const std:: result->segmentedObjects.reserve(keep_idx.size()); for (auto idx : keep_idx) { if (postprocess_semantic_masks) { - all_detections_ptrs[idx].get().mask = segm_postprocess(all_detections_ptrs[idx], all_detections_ptrs[idx].get().mask, - image_size.height, image_size.width); + all_detections_ptrs[idx].get().mask = segm_postprocess(all_detections_ptrs[idx], + all_detections_ptrs[idx].get().mask, + image_size.height, + image_size.width); } result->segmentedObjects.push_back(all_detections_ptrs[idx]); } @@ -112,7 +119,8 @@ std::unique_ptr InstanceSegmentationTiler::merge_results(const std:: if (tiles_results.size()) { auto* iseg_res = static_cast(tiles_results.begin()->get()); if (iseg_res->feature_vector) { - result->feature_vector = ov::Tensor(iseg_res->feature_vector.get_element_type(), iseg_res->feature_vector.get_shape()); + result->feature_vector = + ov::Tensor(iseg_res->feature_vector.get_element_type(), iseg_res->feature_vector.get_shape()); } } @@ -141,8 +149,10 @@ std::unique_ptr InstanceSegmentationTiler::merge_results(const std:: return retVal; } -std::vector> InstanceSegmentationTiler::merge_saliency_maps(const std::vector>& tiles_results, - const cv::Size& image_size, const std::vector& tile_coords) { +std::vector> InstanceSegmentationTiler::merge_saliency_maps( + const std::vector>& tiles_results, + const cv::Size& image_size, + const std::vector& tile_coords) { std::vector>> all_saliecy_maps; all_saliecy_maps.reserve(tiles_results.size()); for (const auto& result : tiles_results) { @@ -186,8 +196,7 @@ std::vector> InstanceSegmentationTiler::merge_saliency_ma if (cv::sum(merged_map[class_idx]) == cv::Scalar(0.)) { merged_map[class_idx] = cv::Mat_(); } - } - else { + } else { cv::resize(image_map_cls, image_map_cls, image_size); cv::Mat(cv::max(merged_map[class_idx], image_map_cls)).copyTo(merged_map[class_idx]); } diff --git a/model_api/cpp/tilers/src/semantic_segmentation.cpp b/model_api/cpp/tilers/src/semantic_segmentation.cpp index c9bed98c..60d58514 100644 --- a/model_api/cpp/tilers/src/semantic_segmentation.cpp +++ b/model_api/cpp/tilers/src/semantic_segmentation.cpp @@ -14,13 +14,13 @@ // limitations under the License. */ +#include +#include +#include -#include #include +#include -#include -#include -#include #include "utils/common.hpp" namespace { @@ -41,46 +41,54 @@ void normalize_soft_prediction(cv::Mat& soft_prediction, const cv::Mat& normaliz } } } -} +} // namespace -SemanticSegmentationTiler::SemanticSegmentationTiler(std::shared_ptr _model, const ov::AnyMap& configuration) : - TilerBase(_model, configuration) { +SemanticSegmentationTiler::SemanticSegmentationTiler(std::shared_ptr _model, + const ov::AnyMap& configuration) + : TilerBase(_model, configuration) { ov::AnyMap extra_config; try { auto ov_model = model->getModel(); extra_config = ov_model->get_rt_info("model_info"); - } - catch (const std::runtime_error&) { + } catch (const std::runtime_error&) { extra_config = model->getInferenceAdapter()->getModelConfig(); } blur_strength = get_from_any_maps("blur_strength", configuration, extra_config, blur_strength); soft_threshold = get_from_any_maps("soft_threshold", configuration, extra_config, soft_threshold); - return_soft_prediction = get_from_any_maps("return_soft_prediction", configuration, extra_config, return_soft_prediction); + return_soft_prediction = + get_from_any_maps("return_soft_prediction", configuration, extra_config, return_soft_prediction); } std::unique_ptr SemanticSegmentationTiler::run(const ImageInputData& inputData) { auto result = this->run_impl(inputData); - return std::unique_ptr(static_cast(result.release())); + return std::unique_ptr( + static_cast(result.release())); } -std::unique_ptr SemanticSegmentationTiler::postprocess_tile(std::unique_ptr tile_result, const cv::Rect&) { +std::unique_ptr SemanticSegmentationTiler::postprocess_tile(std::unique_ptr tile_result, + const cv::Rect&) { ImageResultWithSoftPrediction* soft = dynamic_cast(tile_result.get()); if (!soft) { - throw std::runtime_error("SemanticSegmentationTiler requires the underlying model to return ImageResultWithSoftPrediction"); + throw std::runtime_error( + "SemanticSegmentationTiler requires the underlying model to return ImageResultWithSoftPrediction"); } return tile_result; } -std::unique_ptr SemanticSegmentationTiler::merge_results(const std::vector>& tiles_results, - const cv::Size& image_size, const std::vector& tile_coords) { +std::unique_ptr SemanticSegmentationTiler::merge_results( + const std::vector>& tiles_results, + const cv::Size& image_size, + const std::vector& tile_coords) { if (tiles_results.empty()) { return std::unique_ptr(new ImageResultWithSoftPrediction()); } cv::Mat voting_mask(cv::Size(image_size.width, image_size.height), CV_32SC1, cv::Scalar(0)); auto* sseg_res = static_cast(tiles_results[0].get()); - cv::Mat merged_soft_prediction(cv::Size(image_size.width, image_size.height), CV_32FC(sseg_res->soft_prediction.channels()), cv::Scalar(0)); + cv::Mat merged_soft_prediction(cv::Size(image_size.width, image_size.height), + CV_32FC(sseg_res->soft_prediction.channels()), + cv::Scalar(0)); for (size_t i = 0; i < tiles_results.size(); ++i) { auto* sseg_res = static_cast(tiles_results[i].get()); @@ -90,7 +98,8 @@ std::unique_ptr SemanticSegmentationTiler::merge_results(const std:: normalize_soft_prediction(merged_soft_prediction, voting_mask); - cv::Mat hard_prediction = create_hard_prediction_from_soft_prediction(merged_soft_prediction, soft_threshold, blur_strength); + cv::Mat hard_prediction = + create_hard_prediction_from_soft_prediction(merged_soft_prediction, soft_threshold, blur_strength); std::unique_ptr retVal; if (return_soft_prediction) { @@ -98,8 +107,7 @@ std::unique_ptr SemanticSegmentationTiler::merge_results(const std:: retVal = std::unique_ptr(result); result->soft_prediction = merged_soft_prediction; result->resultImage = hard_prediction; - } - else { + } else { auto* result = new ImageResult(); retVal = std::unique_ptr(result); result->resultImage = hard_prediction; diff --git a/model_api/cpp/tilers/src/tiler_base.cpp b/model_api/cpp/tilers/src/tiler_base.cpp index 5c6df154..889e79e0 100644 --- a/model_api/cpp/tilers/src/tiler_base.cpp +++ b/model_api/cpp/tilers/src/tiler_base.cpp @@ -14,24 +14,24 @@ // limitations under the License. */ -#include -#include - -#include -#include -#include #include +#include +#include +#include +#include +#include -TilerBase::TilerBase(const std::shared_ptr& _model, const ov::AnyMap& configuration, ExecutionMode exec_mode) : - model(_model), run_mode(exec_mode) { - +TilerBase::TilerBase(const std::shared_ptr& _model, + const ov::AnyMap& configuration, + ExecutionMode exec_mode) + : model(_model), + run_mode(exec_mode) { ov::AnyMap extra_config; try { auto ov_model = model->getModel(); extra_config = ov_model->get_rt_info("model_info"); - } - catch (const std::runtime_error&) { + } catch (const std::runtime_error&) { extra_config = model->getInferenceAdapter()->getModelConfig(); } @@ -59,8 +59,7 @@ std::vector TilerBase::tile(const cv::Size& image_size) { if (tile_with_full_img) { coords.reserve(num_h_tiles * num_w_tiles + 1); coords.push_back(cv::Rect(0, 0, image_size.width, image_size.height)); - } - else { + } else { coords.reserve(num_h_tiles * num_w_tiles); } @@ -69,9 +68,10 @@ std::vector TilerBase::tile(const cv::Size& image_size) { int loc_h = static_cast(j * tile_step); int loc_w = static_cast(i * tile_step); - coords.push_back(cv::Rect(loc_w, loc_h, - std::min(static_cast(tile_size), image_size.width - loc_w), - std::min(static_cast(tile_size), image_size.height - loc_h))); + coords.push_back(cv::Rect(loc_w, + loc_h, + std::min(static_cast(tile_size), image_size.width - loc_w), + std::min(static_cast(tile_size), image_size.height - loc_h))); } } return coords; diff --git a/model_api/cpp/utils/include/utils/args_helper.hpp b/model_api/cpp/utils/include/utils/args_helper.hpp index a8aaa283..d57a0947 100644 --- a/model_api/cpp/utils/include/utils/args_helper.hpp +++ b/model_api/cpp/utils/include/utils/args_helper.hpp @@ -10,14 +10,12 @@ #pragma once #include +#include +#include #include #include #include -#include -#include - - std::vector split(const std::string& s, char delim); std::vector parseDevices(const std::string& device_string); @@ -29,8 +27,11 @@ std::map parseLayoutString(const std::string& layout_st std::string formatLayouts(const std::map& layouts); -template -Type get_from_any_maps(const std::string& key, const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority, Type low_priority) { +template +Type get_from_any_maps(const std::string& key, + const ov::AnyMap& top_priority, + const ov::AnyMap& mid_priority, + Type low_priority) { auto topk_iter = top_priority.find(key); if (topk_iter != top_priority.end()) { return topk_iter->second.as(); @@ -42,8 +43,11 @@ Type get_from_any_maps(const std::string& key, const ov::AnyMap& top_priority, c return low_priority; } -template<> -inline bool get_from_any_maps(const std::string& key, const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority, bool low_priority) { +template <> +inline bool get_from_any_maps(const std::string& key, + const ov::AnyMap& top_priority, + const ov::AnyMap& mid_priority, + bool low_priority) { auto topk_iter = top_priority.find(key); if (topk_iter != top_priority.end()) { const std::string& val = topk_iter->second.as(); diff --git a/model_api/cpp/utils/include/utils/async_infer_queue.hpp b/model_api/cpp/utils/include/utils/async_infer_queue.hpp index b1a8f0a4..1dd38aa4 100644 --- a/model_api/cpp/utils/include/utils/async_infer_queue.hpp +++ b/model_api/cpp/utils/include/utils/async_infer_queue.hpp @@ -16,12 +16,10 @@ #include #include +#include #include #include -#include - - class AsyncInferQueue { public: AsyncInferQueue() = default; @@ -32,14 +30,16 @@ class AsyncInferQueue { size_t get_idle_request_id(); void wait_all(); void set_default_callbacks(); - void set_custom_callbacks(std::function callback_args)> f_callback); + void set_custom_callbacks( + std::function callback_args)> f_callback); size_t size() const; void start_async(const ov::Tensor& input, std::shared_ptr userdata = nullptr); void start_async(const std::map& input, std::shared_ptr userdata = nullptr); ov::InferRequest operator[](size_t i); // AsyncInferQueue is not the sole owner of infer requests, although it calls create_infer_request() method. - // ov::InferRequest contains a shared pointer internally, and therefore requests can be copied by clients of AsyncInferQueue. + // ov::InferRequest contains a shared pointer internally, and therefore requests can be copied by clients of + // AsyncInferQueue. protected: std::vector m_requests; std::queue m_idle_handles; diff --git a/model_api/cpp/utils/include/utils/common.hpp b/model_api/cpp/utils/include/utils/common.hpp index 23cbb300..5e4db57e 100644 --- a/model_api/cpp/utils/include/utils/common.hpp +++ b/model_api/cpp/utils/include/utils/common.hpp @@ -10,15 +10,15 @@ #pragma once #include +#include #include #include #include -#include #include "utils/slog.hpp" template -constexpr std::size_t arraySize(const T(&)[N]) noexcept { +constexpr std::size_t arraySize(const T (&)[N]) noexcept { return N; } diff --git a/model_api/cpp/utils/include/utils/image_utils.h b/model_api/cpp/utils/include/utils/image_utils.h index 97de7690..a3641e7f 100644 --- a/model_api/cpp/utils/include/utils/image_utils.h +++ b/model_api/cpp/utils/include/utils/image_utils.h @@ -29,16 +29,25 @@ enum RESIZE_MODE { inline std::string formatResizeMode(RESIZE_MODE mode) { switch (mode) { - case RESIZE_FILL: return "standard"; - case RESIZE_KEEP_ASPECT: return "fit_to_window"; - case RESIZE_KEEP_ASPECT_LETTERBOX: return "fit_to_window_letterbox"; - case RESIZE_CROP: return "crop"; - default: return "unknown"; + case RESIZE_FILL: + return "standard"; + case RESIZE_KEEP_ASPECT: + return "fit_to_window"; + case RESIZE_KEEP_ASPECT_LETTERBOX: + return "fit_to_window_letterbox"; + case RESIZE_CROP: + return "crop"; + default: + return "unknown"; } } -cv::Mat resizeImageExt(const cv::Mat& mat, int width, int height, RESIZE_MODE resizeMode = RESIZE_FILL, - cv::InterpolationFlags interpolationMode = cv::INTER_LINEAR, cv::Rect* roi = nullptr, +cv::Mat resizeImageExt(const cv::Mat& mat, + int width, + int height, + RESIZE_MODE resizeMode = RESIZE_FILL, + cv::InterpolationFlags interpolationMode = cv::INTER_LINEAR, + cv::Rect* roi = nullptr, cv::Scalar BorderConstant = cv::Scalar(0, 0, 0)); ov::preprocess::PostProcessSteps::CustomPostprocessOp createResizeGraph(RESIZE_MODE resizeMode, diff --git a/model_api/cpp/utils/include/utils/kuhn_munkres.hpp b/model_api/cpp/utils/include/utils/kuhn_munkres.hpp index 9fab0ba7..4423d03c 100644 --- a/model_api/cpp/utils/include/utils/kuhn_munkres.hpp +++ b/model_api/cpp/utils/include/utils/kuhn_munkres.hpp @@ -4,11 +4,10 @@ #pragma once -#include "opencv2/core.hpp" - #include #include +#include "opencv2/core.hpp" /// /// \brief The KuhnMunkres class @@ -31,7 +30,7 @@ class KuhnMunkres { /// \return Optimal column index for each row. -1 means that there is no /// column for row. /// - std::vector Solve(const cv::Mat &dissimilarity_matrix); + std::vector Solve(const cv::Mat& dissimilarity_matrix); private: static constexpr int kStar = 1; diff --git a/model_api/cpp/utils/include/utils/nms.hpp b/model_api/cpp/utils/include/utils/nms.hpp index 6a7a2b22..725d4f80 100644 --- a/model_api/cpp/utils/include/utils/nms.hpp +++ b/model_api/cpp/utils/include/utils/nms.hpp @@ -16,10 +16,11 @@ #pragma once -#include "opencv2/core.hpp" #include #include +#include "opencv2/core.hpp" + struct Anchor { float left; float top; @@ -27,8 +28,11 @@ struct Anchor { float bottom; Anchor() = default; - Anchor(float _left, float _top, float _right, float _bottom) : - left(_left), top(_top), right(_right), bottom(_bottom) {} + Anchor(float _left, float _top, float _right, float _bottom) + : left(_left), + top(_top), + right(_right), + bottom(_bottom) {} float getWidth() const { return (right - left) + 1.0f; @@ -48,25 +52,34 @@ struct AnchorLabeled : public Anchor { int labelID = -1; AnchorLabeled() = default; - AnchorLabeled(float _left, float _top, float _right, float _bottom, int _labelID) : - Anchor(_left, _top, _right, _bottom), labelID(_labelID) {} + AnchorLabeled(float _left, float _top, float _right, float _bottom, int _labelID) + : Anchor(_left, _top, _right, _bottom), + labelID(_labelID) {} }; template -std::vector nms(const std::vector& boxes, const std::vector& scores, const float thresh, bool includeBoundaries=false, size_t keep_top_k=0) { +std::vector nms(const std::vector& boxes, + const std::vector& scores, + const float thresh, + bool includeBoundaries = false, + size_t keep_top_k = 0) { if (keep_top_k == 0) { keep_top_k = boxes.size(); } std::vector areas(boxes.size()); for (size_t i = 0; i < boxes.size(); ++i) { - areas[i] = (boxes[i].right - boxes[i].left + includeBoundaries) * (boxes[i].bottom - boxes[i].top + includeBoundaries); + areas[i] = + (boxes[i].right - boxes[i].left + includeBoundaries) * (boxes[i].bottom - boxes[i].top + includeBoundaries); } std::vector order(scores.size()); std::iota(order.begin(), order.end(), 0); - std::sort(order.begin(), order.end(), [&scores](int o1, int o2) { return scores[o1] > scores[o2]; }); + std::sort(order.begin(), order.end(), [&scores](int o1, int o2) { + return scores[o1] > scores[o2]; + }); size_t ordersNum = 0; - for (; ordersNum < order.size() && scores[order[ordersNum]] >= 0 && ordersNum < keep_top_k; ordersNum++); + for (; ordersNum < order.size() && scores[order[ordersNum]] >= 0 && ordersNum < keep_top_k; ordersNum++) + ; std::vector keep; bool shouldContinue = true; @@ -79,9 +92,12 @@ std::vector nms(const std::vector& boxes, const std::vector= 0) { shouldContinue = true; - float overlappingWidth = std::fminf(boxes[idx1].right, boxes[idx2].right) - std::fmaxf(boxes[idx1].left, boxes[idx2].left); - float overlappingHeight = std::fminf(boxes[idx1].bottom, boxes[idx2].bottom) - std::fmaxf(boxes[idx1].top, boxes[idx2].top); - float intersection = overlappingWidth > 0 && overlappingHeight > 0 ? overlappingWidth * overlappingHeight : 0; + float overlappingWidth = std::fminf(boxes[idx1].right, boxes[idx2].right) - + std::fmaxf(boxes[idx1].left, boxes[idx2].left); + float overlappingHeight = std::fminf(boxes[idx1].bottom, boxes[idx2].bottom) - + std::fmaxf(boxes[idx1].top, boxes[idx2].top); + float intersection = + overlappingWidth > 0 && overlappingHeight > 0 ? overlappingWidth * overlappingHeight : 0; float union_area = areas[idx1] + areas[idx2] - intersection; if (0.0f == union_area || intersection / union_area > thresh) { order[j] = -1; @@ -93,5 +109,8 @@ std::vector nms(const std::vector& boxes, const std::vector multiclass_nms(const std::vector& boxes, const std::vector& scores, - const float iou_threshold=0.45f, bool includeBoundaries=false, size_t maxNum=200); +std::vector multiclass_nms(const std::vector& boxes, + const std::vector& scores, + const float iou_threshold = 0.45f, + bool includeBoundaries = false, + size_t maxNum = 200); diff --git a/model_api/cpp/utils/include/utils/ocv_common.hpp b/model_api/cpp/utils/include/utils/ocv_common.hpp index c00694f7..3a25f755 100644 --- a/model_api/cpp/utils/include/utils/ocv_common.hpp +++ b/model_api/cpp/utils/include/utils/ocv_common.hpp @@ -27,19 +27,24 @@ static inline ov::Tensor wrapMat2Tensor(const cv::Mat& mat) { const size_t strideH = mat.step.buf[0]; const size_t strideW = mat.step.buf[1]; - const bool isDense = !isMatFloat ? (strideW == channels && strideH == channels * width) : - (strideW == channels * sizeof(float) && strideH == channels * width * sizeof(float)); + const bool isDense = !isMatFloat + ? (strideW == channels && strideH == channels * width) + : (strideW == channels * sizeof(float) && strideH == channels * width * sizeof(float)); if (!isDense) { throw std::runtime_error("Doesn't support conversion from not dense cv::Mat"); } auto precision = isMatFloat ? ov::element::f32 : ov::element::u8; struct SharedMatAllocator { const cv::Mat mat; - void* allocate(size_t bytes, size_t) {return bytes <= mat.rows * mat.step[0] ? mat.data : nullptr;} + void* allocate(size_t bytes, size_t) { + return bytes <= mat.rows * mat.step[0] ? mat.data : nullptr; + } void deallocate(void*, size_t, size_t) {} - bool is_equal(const SharedMatAllocator& other) const noexcept {return this == &other;} + bool is_equal(const SharedMatAllocator& other) const noexcept { + return this == &other; + } }; - return ov::Tensor(precision, ov::Shape{ 1, height, width, channels }, SharedMatAllocator{mat}); + return ov::Tensor(precision, ov::Shape{1, height, width, channels}, SharedMatAllocator{mat}); } struct IntervalCondition { @@ -47,34 +52,35 @@ struct IntervalCondition { using IndexType = size_t; using ConditionChecker = std::function; - template - constexpr IntervalCondition(IndexType i1, IndexType i2, Cond c) : - impl([=](IndexType i0, const ov::PartialShape& shape) { - return c(shape[i0].get_max_length(), shape[i1].get_max_length()) && c(shape[i0].get_max_length(), shape[i2].get_max_length());}) - {} - bool operator() (IndexType i0, const ov::PartialShape& shape) const { return impl(i0, shape); } + template + constexpr IntervalCondition(IndexType i1, IndexType i2, Cond c) + : impl([=](IndexType i0, const ov::PartialShape& shape) { + return c(shape[i0].get_max_length(), shape[i1].get_max_length()) && + c(shape[i0].get_max_length(), shape[i2].get_max_length()); + }) {} + bool operator()(IndexType i0, const ov::PartialShape& shape) const { + return impl(i0, shape); + } + private: ConditionChecker impl; }; -template class Cond, class ...Args> -IntervalCondition makeCond(Args&&...args) { +template