Segmentation tiling

src/cpp/include/tasks/instance_segmentation.h

@@ -16,40 +16,75 @@
 class InstanceSegmentation {
 public:
     std::shared_ptr<InferenceAdapter> adapter;
-    VisionPipeline<InstanceSegmentationResult> pipeline;
-
-    InstanceSegmentation(std::shared_ptr<InferenceAdapter> adapter) : adapter(adapter) {
-        pipeline = VisionPipeline<InstanceSegmentationResult>(
-            adapter,
-            [&](cv::Mat image) {
-                return preprocess(image);
-            },
-            [&](InferenceResult result) {
-                return postprocess(result);
-            });
+    std::unique_ptr<Pipeline<InstanceSegmentationResult>> pipeline;
 
+    InstanceSegmentation(std::shared_ptr<InferenceAdapter> adapter, const ov::AnyMap& configuration)
+        : adapter(adapter) {
         auto config = adapter->getModelConfig();
+        tiling = utils::get_from_any_maps("tiling", configuration, config, tiling);
+        if (tiling) {
+            pipeline = std::make_unique<TilingPipeline<InstanceSegmentationResult>>(
+                adapter,
+                utils::get_tiling_info_from_config(config),
+                [&](cv::Mat image) {
+                    return preprocess(image);
+                },
+                [&](InferenceResult result) {
+                    return postprocess(result);
+                },
+                [&](InstanceSegmentationResult result, const cv::Rect& coord) {
+                    return postprocess_tile(result, coord);
+                },
+                [&](const std::vector<InstanceSegmentationResult>& tiles_results,
+                    const cv::Size& image_size,
+                    const std::vector<cv::Rect>& tile_coords,
+                    const utils::TilingInfo& tiling_info) {
+                    return merge_tiling_results(tiles_results, image_size, tile_coords, tiling_info);
+                });
+        } else {
+            pipeline = std::make_unique<VisionPipeline<InstanceSegmentationResult>>(
+                adapter,
+                [&](cv::Mat image) {
+                    return preprocess(image);
+                },
+                [&](InferenceResult result) {
+                    return postprocess(result);
+                });
+        }
         labels = utils::get_from_any_maps("labels", config, {}, labels);
         confidence_threshold = utils::get_from_any_maps("confidence_threshold", config, {}, confidence_threshold);
         input_shape.width = utils::get_from_any_maps("orig_width", config, {}, input_shape.width);
         input_shape.height = utils::get_from_any_maps("orig_height", config, {}, input_shape.width);
+        resize_mode = utils::get_from_any_maps("resize_type", config, {}, resize_mode);
     }
 
     static void serialize(std::shared_ptr<ov::Model>& ov_model);
-    static InstanceSegmentation load(const std::string& model_path);
+    static InstanceSegmentation load(const std::string& model_path, const ov::AnyMap& configuration);
 
     InstanceSegmentationResult infer(cv::Mat image);
     std::vector<InstanceSegmentationResult> inferBatch(std::vector<cv::Mat> image);
 
     std::map<std::string, ov::Tensor> preprocess(cv::Mat);
     InstanceSegmentationResult postprocess(InferenceResult& infResult);
+    InstanceSegmentationResult postprocess_tile(InstanceSegmentationResult, const cv::Rect&);
+    InstanceSegmentationResult merge_tiling_results(const std::vector<InstanceSegmentationResult>& tiles_results,
+                                                    const cv::Size& image_size,
+                                                    const std::vector<cv::Rect>& tile_coords,
+                                                    const utils::TilingInfo& tiling_info);
+    std::vector<cv::Mat_<std::uint8_t>> merge_saliency_maps(const std::vector<InstanceSegmentationResult>&,
+                                                            const cv::Size&,
+                                                            const std::vector<cv::Rect>&,
+                                                            const utils::TilingInfo&);
 
     static std::vector<SegmentedObjectWithRects> getRotatedRectangles(const InstanceSegmentationResult& result);
     static std::vector<Contour> getContours(const std::vector<SegmentedObject>& objects);
 
     bool postprocess_semantic_masks = true;
 
 private:
+    bool tiling;
+
+    utils::RESIZE_MODE resize_mode;
     std::vector<std::string> labels;
     std::string getLabelName(size_t labelID) {
         return labelID < labels.size() ? labels[labelID] : std::string("Label #") + std::to_string(labelID);

src/cpp/include/tasks/semantic_segmentation.h

@@ -11,37 +11,66 @@
 #include "tasks/results.h"
 #include "utils/config.h"
 #include "utils/preprocessing.h"
+#include "utils/tiling.h"
 #include "utils/vision_pipeline.h"
 
 class SemanticSegmentation {
 public:
-    VisionPipeline<SemanticSegmentationResult> pipeline;
+    std::unique_ptr<Pipeline<SemanticSegmentationResult>> pipeline;
     std::shared_ptr<InferenceAdapter> adapter;
-    SemanticSegmentation(std::shared_ptr<InferenceAdapter> adapter) : adapter(adapter) {
-        pipeline = VisionPipeline<SemanticSegmentationResult>(
-            adapter,
-            [&](cv::Mat image) {
-                return preprocess(image);
-            },
-            [&](InferenceResult result) {
-                return postprocess(result);
-            });
-
+    SemanticSegmentation(std::shared_ptr<InferenceAdapter> adapter, const ov::AnyMap& configuration)
+        : adapter(adapter) {
         auto config = adapter->getModelConfig();
+        tiling = utils::get_from_any_maps("tiling", configuration, config, tiling);
+        if (tiling) {
+            pipeline = std::make_unique<TilingPipeline<SemanticSegmentationResult>>(
+                adapter,
+                utils::get_tiling_info_from_config(config),
+                [&](cv::Mat image) {
+                    return preprocess(image);
+                },
+                [&](InferenceResult result) {
+                    return postprocess(result);
+                },
+                [&](SemanticSegmentationResult& result, const cv::Rect& coord) {
+                    return postprocess_tile(result, coord);
+                },
+                [&](const std::vector<SemanticSegmentationResult>& tiles_results,
+                    const cv::Size& image_size,
+                    const std::vector<cv::Rect>& tile_coords,
+                    const utils::TilingInfo& tiling_info) {
+                    return merge_tiling_results(tiles_results, image_size, tile_coords, tiling_info);
+                });
+        } else {
+            pipeline = std::make_unique<VisionPipeline<SemanticSegmentationResult>>(
+                adapter,
+                [&](cv::Mat image) {
+                    return preprocess(image);
+                },
+                [&](InferenceResult result) {
+                    return postprocess(result);
+                });
+        }
+
         labels = utils::get_from_any_maps("labels", config, {}, labels);
         soft_threshold = utils::get_from_any_maps("soft_threshold", config, {}, soft_threshold);
         blur_strength = utils::get_from_any_maps("blur_strength", config, {}, blur_strength);
     }
 
     static void serialize(std::shared_ptr<ov::Model>& ov_model);
-    static SemanticSegmentation load(const std::string& model_path);
+    static SemanticSegmentation load(const std::string& model_path, const ov::AnyMap& configuration = {});
 
     std::map<std::string, ov::Tensor> preprocess(cv::Mat);
     SemanticSegmentationResult postprocess(InferenceResult& infResult);
     std::vector<Contour> getContours(const SemanticSegmentationResult& result);
 
     SemanticSegmentationResult infer(cv::Mat image);
     std::vector<SemanticSegmentationResult> inferBatch(std::vector<cv::Mat> image);
+    SemanticSegmentationResult postprocess_tile(SemanticSegmentationResult, const cv::Rect&);
+    SemanticSegmentationResult merge_tiling_results(const std::vector<SemanticSegmentationResult>& tiles_results,
+                                                    const cv::Size& image_size,
+                                                    const std::vector<cv::Rect>& tile_coords,
+                                                    const utils::TilingInfo& tiling_info);
 
 private:
     cv::Mat create_hard_prediction_from_soft_prediction(cv::Mat, float threshold, int blur_strength);
@@ -50,6 +79,7 @@ class SemanticSegmentation {
     int blur_strength = -1;
     float soft_threshold = -std::numeric_limits<float>::infinity();
     bool return_soft_prediction = true;
+    bool tiling = false;
 
     std::vector<std::string> labels;
 

src/cpp/include/utils/config.h

@@ -8,6 +8,14 @@
 #include <opencv2/opencv.hpp>
 #include <openvino/openvino.hpp>
 namespace utils {
+enum RESIZE_MODE {
+    RESIZE_FILL,
+    RESIZE_KEEP_ASPECT,
+    RESIZE_KEEP_ASPECT_LETTERBOX,
+    RESIZE_CROP,
+    NO_RESIZE,
+};
+
 template <typename Type>
 Type get_from_any_maps(const std::string& key,
                        const ov::AnyMap& top_priority,
@@ -42,6 +50,29 @@ inline bool get_from_any_maps(const std::string& key,
     return low_priority;
 }
 
+template <>
+inline RESIZE_MODE get_from_any_maps(const std::string& key,
+                                     const ov::AnyMap& top_priority,
+                                     const ov::AnyMap& mid_priority,
+                                     RESIZE_MODE low_priority) {
+    std::string resize_type = "standard";
+    resize_type = utils::get_from_any_maps("resize_type", top_priority, mid_priority, resize_type);
+    RESIZE_MODE resize = RESIZE_FILL;
+    if ("crop" == resize_type) {
+        resize = RESIZE_CROP;
+    } else if ("standard" == resize_type) {
+        resize = RESIZE_FILL;
+    } else if ("fit_to_window" == resize_type) {
+        resize = RESIZE_KEEP_ASPECT;
+    } else if ("fit_to_window_letterbox" == resize_type) {
+        resize = RESIZE_KEEP_ASPECT_LETTERBOX;
+    } else {
+        throw std::runtime_error("Unknown value for resize_type arg");
+    }
+
+    return resize;
+}
+
 inline bool model_has_embedded_processing(std::shared_ptr<ov::Model> model) {
     if (model->has_rt_info("model_info")) {
         auto model_info = model->get_rt_info<ov::AnyMap>("model_info");

src/cpp/include/utils/preprocessing.h

@@ -8,15 +8,9 @@
 #include <opencv2/opencv.hpp>
 #include <openvino/openvino.hpp>
 
-namespace utils {
-enum RESIZE_MODE {
-    RESIZE_FILL,
-    RESIZE_KEEP_ASPECT,
-    RESIZE_KEEP_ASPECT_LETTERBOX,
-    RESIZE_CROP,
-    NO_RESIZE,
-};
+#include "utils/config.h"
 
+namespace utils {
 std::shared_ptr<ov::Model> embedProcessing(std::shared_ptr<ov::Model>& model,
                                            const std::string& inputName,
                                            const ov::Layout&,

src/cpp/include/utils/tiling.h

@@ -8,6 +8,8 @@
 #include <opencv2/opencv.hpp>
 #include <openvino/openvino.hpp>
 
+#include "utils/config.h"
+
 namespace utils {
 
 struct TilingInfo {
@@ -24,30 +26,10 @@ inline bool config_contains_tiling_info(const ov::AnyMap& config) {
 
 inline TilingInfo get_tiling_info_from_config(const ov::AnyMap& config) {
     TilingInfo info;
-    {
-        auto iter = config.find("tile_size");
-        if (iter != config.end()) {
-            info.tile_size = iter->second.as<size_t>();
-        }
-    }
-    {
-        auto iter = config.find("tiles_overlap");
-        if (iter != config.end()) {
-            info.tiles_overlap = iter->second.as<float>();
-        }
-    }
-    {
-        auto iter = config.find("iou_threshold");
-        if (iter != config.end()) {
-            info.iou_threshold = iter->second.as<float>();
-        }
-    }
-    {
-        auto iter = config.find("tile_with_full_img");
-        if (iter != config.end()) {
-            info.tile_with_full_image = iter->second.as<bool>();
-        }
-    }
+    info.tile_size = utils::get_from_any_maps("tile_size", config, {}, info.tile_size);
+    info.tiles_overlap = utils::get_from_any_maps("tiles_overlap", config, {}, info.tiles_overlap);
+    info.iou_threshold = utils::get_from_any_maps("iou_threshold", config, {}, info.iou_threshold);
+    info.tile_with_full_image = utils::get_from_any_maps("tile_with_full_image", config, {}, info.tile_with_full_image);
     return info;
 }
 

src/cpp/include/utils/vision_pipeline.h

@@ -79,10 +79,10 @@ class TilingPipeline : public Pipeline<ResultType> {
     std::function<InferenceInput(cv::Mat)> preprocess;
     std::function<ResultType(InferenceResult)> postprocess;
     std::function<ResultType(ResultType&, const cv::Rect&)> postprocess_tile;
-    std::function<DetectionResult(const std::vector<DetectionResult>&,
-                                  const cv::Size&,
-                                  const std::vector<cv::Rect>&,
-                                  const utils::TilingInfo&)>
+    std::function<ResultType(const std::vector<ResultType>&,
+                             const cv::Size&,
+                             const std::vector<cv::Rect>&,
+                             const utils::TilingInfo&)>
         merge_tiling_results;
 
 public:
@@ -92,10 +92,10 @@ class TilingPipeline : public Pipeline<ResultType> {
                    std::function<InferenceInput(cv::Mat)> preprocess,
                    std::function<ResultType(InferenceResult)> postprocess,
                    std::function<ResultType(ResultType&, const cv::Rect&)> postprocess_tile,
-                   std::function<DetectionResult(const std::vector<DetectionResult>&,
-                                                 const cv::Size&,
-                                                 const std::vector<cv::Rect>&,
-                                                 const utils::TilingInfo&)> merge_tiling_results)
+                   std::function<ResultType(const std::vector<ResultType>&,
+                                            const cv::Size&,
+                                            const std::vector<cv::Rect>&,
+                                            const utils::TilingInfo&)> merge_tiling_results)
         : adapter(adapter),
           tiling_info(tiling_info),
           preprocess(preprocess),
@@ -111,7 +111,7 @@ class TilingPipeline : public Pipeline<ResultType> {
             auto tile_img = cv::Mat(image, coord);
             auto input = preprocess(tile_img.clone());
             InferenceResult result;
-            result.inputImageSize = image.size();
+            result.inputImageSize = coord.size();
             result.data = adapter->infer(input);
             auto tile_result = postprocess(result);
             tile_results.push_back(postprocess_tile(tile_result, coord));
@@ -146,7 +146,7 @@ class TilingPipeline : public Pipeline<ResultType> {
                 auto input = preprocess(tile_img.clone());
                 auto additional_data = std::make_shared<ov::AnyMap>();
                 additional_data->insert({"index", i});
-                additional_data->insert({"inputImageSize", images[i].size()});
+                additional_data->insert({"inputImageSize", coord.size()});
                 additional_data->insert({"tileCoord", coord});
                 adapter->inferAsync(input, additional_data);
             }