Refactor - Implement Classification (#304)

* Remove cpp implementation

* Move in code from POC

* Satisfy pre-commit checks

* Remove precommit tests for cpp

* Attempt to fix serving_api tests

* Use old InputParser for test_accuracy

* Remove async from public api side

* Add and update license to 2025

* Remove empty line between template and class

* Start implementing classification model

Added multiclass classification

* Implement multilabel post processing

* Implement hierarchical and resolvers

* Enable classification in accuracy tests

* Remove rogue cout

* Implement classification serialization WIP

* Fix classification serialization

Had to set ov::batch(model, 1);

* Working serialization for instance and semantic segmentation

* Attempt openvino 2025.1 for failing test

* Add specific implementation for get_from_any_map for bool

Sometimes the value for bool is "True"
This evaluation from string is only added in openvino 2025.1.0

* Remove input shape as input for the tasks

orig_width and height are stored in rt_info anyway.
Serialization is a separate step and it _should_ output the same model

* Satisfy clang

Author: RHeckerIntel

src/cpp/CMakeLists.txt

@@ -8,6 +8,14 @@ project(vision_api
         HOMEPAGE_URL "https://github.com/openvinotoolkit/model_api/"
         LANGUAGES CXX C)
 
+include(FetchContent)
+FetchContent_Declare(json GIT_REPOSITORY https://github.com/nlohmann/json.git
+                          GIT_TAG d41ca94fa85d5119852e2f7a3f94335cc7cb0486  # PR #4709, fixes cmake deprecation warnings
+                    )
+
+FetchContent_MakeAvailable(json)
+
+
 find_package(OpenCV REQUIRED COMPONENTS core imgproc)
 
 find_package(OpenVINO REQUIRED
@@ -20,5 +28,5 @@ file(GLOB ADAPTERS_SOURCES src/adapters/*.cpp)
 
 add_library(model_api STATIC ${TASK_SOURCES} ${TASKS_SOURCES} ${UTILS_SOURCES} ${ADAPTERS_SOURCES} ${TILERS_SOURCES})
 
-target_link_libraries(model_api PUBLIC openvino::runtime opencv_core opencv_imgproc)
+target_link_libraries(model_api PUBLIC openvino::runtime opencv_core opencv_imgproc PRIVATE nlohmann_json::nlohmann_json)
 target_include_directories(model_api PUBLIC ${PROJECT_SOURCE_DIR}/include)

src/cpp/include/tasks/classification.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2020-2025 Intel Corporation
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <opencv2/opencv.hpp>
+#include <openvino/openvino.hpp>
+
+#include "adapters/inference_adapter.h"
+#include "tasks/classification/resolvers.h"
+#include "tasks/results.h"
+#include "utils/config.h"
+#include "utils/vision_pipeline.h"
+
+class Classification {
+public:
+    std::shared_ptr<InferenceAdapter> adapter;
+    VisionPipeline<ClassificationResult> pipeline;
+
+    Classification(std::shared_ptr<InferenceAdapter> adapter) : adapter(adapter) {
+        pipeline = VisionPipeline<ClassificationResult>(
+            adapter,
+            [&](cv::Mat image) {
+                return preprocess(image);
+            },
+            [&](InferenceResult result) {
+                return postprocess(result);
+            });
+
+        auto config = adapter->getModelConfig();
+        labels = utils::get_from_any_maps("labels", config, {}, labels);
+
+        topk = utils::get_from_any_maps("topk", config, {}, topk);
+        multilabel = utils::get_from_any_maps("multilabel", config, {}, multilabel);
+        output_raw_scores = utils::get_from_any_maps("output_raw_scores", config, {}, output_raw_scores);
+        confidence_threshold = utils::get_from_any_maps("confidence_threshold", config, {}, confidence_threshold);
+        hierarchical = utils::get_from_any_maps("hierarchical", config, {}, hierarchical);
+        hierarchical_config = utils::get_from_any_maps("hierarchical_config", config, {}, hierarchical_config);
+        hierarchical_postproc = utils::get_from_any_maps("hierarchical_postproc", config, {}, hierarchical_postproc);
+        if (hierarchical) {
+            if (hierarchical_config.empty()) {
+                throw std::runtime_error("Error: empty hierarchical classification config");
+            }
+            hierarchical_info = HierarchicalConfig(hierarchical_config);
+            if (hierarchical_postproc == "probabilistic") {
+                resolver = std::make_unique<ProbabilisticLabelsResolver>(hierarchical_info);
+            } else if (hierarchical_postproc == "greedy") {
+                resolver = std::make_unique<GreedyLabelsResolver>(hierarchical_info);
+            } else {
+                throw std::runtime_error("Wrong hierarchical labels postprocessing type");
+            }
+        }
+    }
+
+    static void serialize(std::shared_ptr<ov::Model>& ov_model);
+    static Classification load(const std::string& model_path);
+
+    ClassificationResult infer(cv::Mat image);
+    std::vector<ClassificationResult> inferBatch(std::vector<cv::Mat> image);
+
+    std::map<std::string, ov::Tensor> preprocess(cv::Mat);
+    ClassificationResult postprocess(InferenceResult& infResult);
+
+    bool postprocess_semantic_masks = true;
+
+private:
+    ClassificationResult get_multilabel_predictions(InferenceResult& infResult, bool add_raw_scores);
+    ClassificationResult get_multiclass_predictions(InferenceResult& infResult, bool add_raw_scores);
+    ClassificationResult get_hierarchical_predictions(InferenceResult& infResult, bool add_raw_scores);
+
+    ov::Tensor reorder_saliency_maps(const ov::Tensor& source_maps);
+
+    // multiclass serialization step
+    static void addOrFindSoftmaxAndTopkOutputs(std::shared_ptr<ov::Model>& model, size_t topk, bool add_raw_scores);
+
+private:
+    cv::Size input_shape;
+    std::vector<std::string> labels;
+    float confidence_threshold = 0.5f;
+
+    bool multilabel = false;
+    bool hierarchical = false;
+    bool output_raw_scores = false;
+
+    // hierarchical
+    size_t topk = 1;
+    std::string hierarchical_config;
+    std::string hierarchical_postproc = "greedy";
+    HierarchicalConfig hierarchical_info;
+    std::unique_ptr<GreedyLabelsResolver> resolver;
+};

src/cpp/include/tasks/classification/resolvers.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2020-2025 Intel Corporation
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+struct HierarchicalConfig {
+    std::map<std::string, int> label_to_idx;
+    std::vector<std::pair<std::string, std::string>> label_tree_edges;
+    std::vector<std::vector<std::string>> all_groups;
+    std::map<size_t, std::pair<size_t, size_t>> head_idx_to_logits_range;
+    std::map<size_t, std::string> logit_idx_to_label;
+    size_t num_multiclass_heads;
+    size_t num_multilabel_heads;
+    size_t num_single_label_classes;
+
+    HierarchicalConfig() = default;
+    HierarchicalConfig(const std::string&);
+};
+
+class SimpleLabelsGraph {
+public:
+    SimpleLabelsGraph() = default;
+    SimpleLabelsGraph(const std::vector<std::string>& vertices_);
+    void add_edge(const std::string& parent, const std::string& child);
+    std::vector<std::string> get_children(const std::string& label) const;
+    std::string get_parent(const std::string& label) const;
+    std::vector<std::string> get_ancestors(const std::string& label) const;
+    std::vector<std::string> get_labels_in_topological_order();
+
+protected:
+    std::vector<std::string> vertices;
+    std::unordered_map<std::string, std::vector<std::string>> adj;
+    std::unordered_map<std::string, std::string> parents_map;
+    bool t_sort_cache_valid = false;
+    std::vector<std::string> topological_order_cache;
+
+    std::vector<std::string> topological_sort();
+};
+
+class GreedyLabelsResolver {
+public:
+    GreedyLabelsResolver() = default;
+    GreedyLabelsResolver(const HierarchicalConfig&);
+
+    virtual std::map<std::string, float> resolve_labels(const std::vector<std::reference_wrapper<std::string>>& labels,
+                                                        const std::vector<float>& scores);
+
+protected:
+    std::map<std::string, int> label_to_idx;
+    std::vector<std::pair<std::string, std::string>> label_relations;
+    std::vector<std::vector<std::string>> label_groups;
+
+    std::string get_parent(const std::string& label);
+    std::vector<std::string> get_predecessors(const std::string& label, const std::vector<std::string>& candidates);
+};
+
+class ProbabilisticLabelsResolver : public GreedyLabelsResolver {
+public:
+    ProbabilisticLabelsResolver() = default;
+    ProbabilisticLabelsResolver(const HierarchicalConfig&);
+
+    virtual std::map<std::string, float> resolve_labels(const std::vector<std::reference_wrapper<std::string>>& labels,
+                                                        const std::vector<float>& scores);
+    std::unordered_map<std::string, float> add_missing_ancestors(const std::unordered_map<std::string, float>&) const;
+    std::map<std::string, float> resolve_exclusive_labels(const std::unordered_map<std::string, float>&) const;
+    void suppress_descendant_output(std::map<std::string, float>&);
+
+protected:
+    SimpleLabelsGraph label_tree;
+};

src/cpp/include/tasks/detection/ssd.h

@@ -28,15 +28,17 @@ class SSD {
 public:
     std::shared_ptr<InferenceAdapter> adapter;
 
-    SSD(std::shared_ptr<InferenceAdapter> adapter, cv::Size input_shape) : adapter(adapter), input_shape(input_shape) {
+    SSD(std::shared_ptr<InferenceAdapter> adapter) : adapter(adapter), input_shape(input_shape) {
         auto config = adapter->getModelConfig();
         labels = utils::get_from_any_maps("labels", config, {}, labels);
         confidence_threshold = utils::get_from_any_maps("confidence_threshold", config, {}, confidence_threshold);
+        input_shape.width = utils::get_from_any_maps("orig_width", config, {}, input_shape.width);
+        input_shape.height = utils::get_from_any_maps("orig_height", config, {}, input_shape.height);
     }
     std::map<std::string, ov::Tensor> preprocess(cv::Mat);
     DetectionResult postprocess(InferenceResult& infResult);
 
-    static cv::Size serialize(std::shared_ptr<ov::Model> ov_model);
+    static void serialize(std::shared_ptr<ov::Model> ov_model);
 
     SSDOutputMode output_mode;
 

src/cpp/include/tasks/instance_segmentation.h

@@ -18,9 +18,7 @@ class InstanceSegmentation {
     std::shared_ptr<InferenceAdapter> adapter;
     VisionPipeline<InstanceSegmentationResult> pipeline;
 
-    InstanceSegmentation(std::shared_ptr<InferenceAdapter> adapter, cv::Size input_shape)
-        : adapter(adapter),
-          input_shape(input_shape) {
+    InstanceSegmentation(std::shared_ptr<InferenceAdapter> adapter) : adapter(adapter) {
         pipeline = VisionPipeline<InstanceSegmentationResult>(
             adapter,
             [&](cv::Mat image) {
@@ -33,9 +31,11 @@ class InstanceSegmentation {
         auto config = adapter->getModelConfig();
         labels = utils::get_from_any_maps("labels", config, {}, labels);
         confidence_threshold = utils::get_from_any_maps("confidence_threshold", config, {}, confidence_threshold);
+        input_shape.width = utils::get_from_any_maps("orig_width", config, {}, input_shape.width);
+        input_shape.height = utils::get_from_any_maps("orig_height", config, {}, input_shape.width);
     }
 
-    static cv::Size serialize(std::shared_ptr<ov::Model>& ov_model);
+    static void serialize(std::shared_ptr<ov::Model>& ov_model);
     static InstanceSegmentation load(const std::string& model_path);
 
     InstanceSegmentationResult infer(cv::Mat image);

src/cpp/include/tasks/results.h

@@ -146,3 +146,50 @@ struct InstanceSegmentationResult {
     std::vector<cv::Mat_<std::uint8_t>> saliency_map;
     ov::Tensor feature_vector;
 };
+
+struct ClassificationResult {
+    friend std::ostream& operator<<(std::ostream& os, const ClassificationResult& prediction) {
+        for (const ClassificationResult::Classification& classification : prediction.topLabels) {
+            os << classification << ", ";
+        }
+        try {
+            os << prediction.saliency_map.get_shape() << ", ";
+        } catch (ov::Exception&) {
+            os << "[0], ";
+        }
+        try {
+            os << prediction.feature_vector.get_shape() << ", ";
+        } catch (ov::Exception&) {
+            os << "[0], ";
+        }
+        try {
+            os << prediction.raw_scores.get_shape();
+        } catch (ov::Exception&) {
+            os << "[0]";
+        }
+        return os;
+    }
+
+    explicit operator std::string() {
+        std::stringstream ss;
+        ss << *this;
+        return ss.str();
+    }
+
+    struct Classification {
+        size_t id;
+        std::string label;
+        float score;
+
+        Classification(size_t id, const std::string& label, float score) : id(id), label(label), score(score) {}
+
+        friend std::ostream& operator<<(std::ostream& os, const Classification& prediction) {
+            return os << prediction.id << " (" << prediction.label << "): " << std::fixed << std::setprecision(3)
+                      << prediction.score;
+        }
+    };
+
+    std::vector<Classification> topLabels;
+    ov::Tensor saliency_map, feature_vector,
+        raw_scores;  // Contains "raw_scores", "saliency_map" and "feature_vector" model outputs if such exist
+};

src/cpp/include/tasks/semantic_segmentation.h

@@ -33,7 +33,7 @@ class SemanticSegmentation {
         blur_strength = utils::get_from_any_maps("blur_strength", config, {}, blur_strength);
     }
 
-    static cv::Size serialize(std::shared_ptr<ov::Model>& ov_model);
+    static void serialize(std::shared_ptr<ov::Model>& ov_model);
     static SemanticSegmentation load(const std::string& model_path);
 
     std::map<std::string, ov::Tensor> preprocess(cv::Mat);

src/cpp/include/utils/config.h

@@ -24,6 +24,24 @@ Type get_from_any_maps(const std::string& key,
     return low_priority;
 }
 
+template <>
+inline bool get_from_any_maps(const std::string& key,
+                              const ov::AnyMap& top_priority,
+                              const ov::AnyMap& mid_priority,
+                              bool low_priority) {
+    auto topk_iter = top_priority.find(key);
+    if (topk_iter != top_priority.end()) {
+        const std::string& val = topk_iter->second.as<std::string>();
+        return val == "True" || val == "YES";
+    }
+    topk_iter = mid_priority.find(key);
+    if (topk_iter != mid_priority.end()) {
+        const std::string& val = topk_iter->second.as<std::string>();
+        return val == "True" || val == "YES";
+    }
+    return low_priority;
+}
+
 inline bool model_has_embedded_processing(std::shared_ptr<ov::Model> model) {
     if (model->has_rt_info("model_info")) {
         auto model_info = model->get_rt_info<ov::AnyMap>("model_info");
@@ -97,6 +115,31 @@ static inline std::tuple<bool, ov::Layout> makeGuesLayoutFrom4DShape(const ov::P
     return {false, ov::Layout{}};
 }
 
+static inline std::map<std::string, ov::Layout> parseLayoutString(const std::string& layout_string) {
+    // Parse parameter string like "input0:NCHW,input1:NC" or "NCHW" (applied to all
+    // inputs)
+    std::map<std::string, ov::Layout> layouts;
+    std::string searchStr =
+        (layout_string.find_last_of(':') == std::string::npos && !layout_string.empty() ? ":" : "") + layout_string;
+    auto colonPos = searchStr.find_last_of(':');
+    while (colonPos != std::string::npos) {
+        auto startPos = searchStr.find_last_of(',');
+        auto inputName = searchStr.substr(startPos + 1, colonPos - startPos - 1);
+        auto inputLayout = searchStr.substr(colonPos + 1);
+        layouts[inputName] = ov::Layout(inputLayout);
+        searchStr.resize(startPos + 1);
+        if (searchStr.empty() || searchStr.back() != ',') {
+            break;
+        }
+        searchStr.pop_back();
+        colonPos = searchStr.find_last_of(':');
+    }
+    if (!searchStr.empty()) {
+        throw std::invalid_argument("Can't parse input layout string: " + layout_string);
+    }
+    return layouts;
+}
+
 static inline ov::Layout getLayoutFromShape(const ov::PartialShape& shape) {
     if (shape.size() == 2) {
         return "NC";
@@ -133,4 +176,22 @@ static inline ov::Layout getLayoutFromShape(const ov::PartialShape& shape) {
     throw std::runtime_error("Usupported " + std::to_string(shape.size()) + "D shape");
 }
 
+static inline ov::Layout getInputLayout(const ov::Output<ov::Node>& input,
+                                        std::map<std::string, ov::Layout>& inputsLayouts) {
+    ov::Layout layout = ov::layout::get_layout(input);
+    if (layout.empty()) {
+        if (inputsLayouts.empty()) {
+            layout = getLayoutFromShape(input.get_partial_shape());
+            std::cout << "Automatically detected layout '" << layout.to_string() << "' for input '"
+                      << input.get_any_name() << "' will be used." << std::endl;
+        } else if (inputsLayouts.size() == 1) {
+            layout = inputsLayouts.begin()->second;
+        } else {
+            layout = inputsLayouts[input.get_any_name()];
+        }
+    }
+
+    return layout;
+}
+
 }  // namespace utils