diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index 61c877ca..f94157b4 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -46,4 +46,4 @@ jobs: make -j - name: Run CPP Test run: | - build/test_accuracy -d data -p tests/python/accuracy/public_scope.json + build/test_accuracy -d data -p tests/cpp/accuracy/public_scope.json diff --git a/examples/cpp/asynchronous_api/main.cpp b/examples/cpp/asynchronous_api/main.cpp index f8edc2cc..0ed44d53 100644 --- a/examples/cpp/asynchronous_api/main.cpp +++ b/examples/cpp/asynchronous_api/main.cpp @@ -53,8 +53,8 @@ int main(int argc, char* argv[]) try { std::cout << "Batch mode inference results:\n"; for (const auto& result : results) { - for (auto& obj : result->objects) { - std::cout << " " << std::left << std::setw(9) << obj.confidence << " " << obj.label << "\n"; + for (auto& obj : result->boxes) { + std::cout << obj << std::endl; } std::cout << std::string(10, '-') << "\n"; } @@ -62,15 +62,14 @@ int main(int argc, char* argv[]) try { std::cout << "Async mode inference results:\n"; // Set callback to grab results once the inference is done - model->setCallback([](std::unique_ptr result, const ov::AnyMap& callback_args) { - auto det_result = std::unique_ptr(static_cast(result.release())); + model->setCallback([](std::unique_ptr result, const ov::AnyMap& callback_args) { // callback_args can contain arbitrary data size_t id = callback_args.find("id")->second.as(); std::cout << "Request with id " << id << " is finished\n"; - for (auto& obj : det_result->objects) { - std::cout << " " << std::left << std::setw(9) << obj.confidence << " " << obj.label << "\n"; + for (auto& obj : result->boxes) { + std::cout << " " << obj << std::endl; } std::cout << std::string(10, '-') << "\n"; }); diff --git a/examples/cpp/synchronous_api/main.cpp b/examples/cpp/synchronous_api/main.cpp index 1f79a035..fc7d2122 100644 --- a/examples/cpp/synchronous_api/main.cpp +++ b/examples/cpp/synchronous_api/main.cpp @@ -6,11 +6,7 @@ #include #include #include -#include - -#include #include -#include #include #include #include @@ -37,10 +33,8 @@ int main(int argc, char* argv[]) try { auto result = model->infer(image); // Process detections - for (auto& obj : result->objects) { - std::cout << " " << std::left << std::setw(9) << obj.label << " | " << std::setw(10) << obj.confidence << " | " - << std::setw(4) << int(obj.x) << " | " << std::setw(4) << int(obj.y) << " | " << std::setw(4) - << int(obj.x + obj.width) << " | " << std::setw(4) << int(obj.y + obj.height) << "\n"; + for (auto& obj : result->boxes) { + std::cout << obj << std::endl; } } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/src/cpp/models/include/models/anomaly_model.h b/src/cpp/models/include/models/anomaly_model.h index 1cc5be22..6dd55d91 100644 --- a/src/cpp/models/include/models/anomaly_model.h +++ b/src/cpp/models/include/models/anomaly_model.h @@ -23,9 +23,9 @@ class AnomalyModel : public BaseModel { const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); - virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch(const std::vector& inputImgs); - std::unique_ptr postprocess(InferenceResult& infResult) override; + virtual std::unique_ptr infer(const ImageInputData& inputData); + virtual std::vector> inferBatch(const std::vector& inputImgs); + std::unique_ptr postprocess(InferenceResult& infResult) override; friend std::ostream& operator<<(std::ostream& os, std::unique_ptr& model); diff --git a/src/cpp/models/include/models/base_model.h b/src/cpp/models/include/models/base_model.h index 85131805..383bc865 100644 --- a/src/cpp/models/include/models/base_model.h +++ b/src/cpp/models/include/models/base_model.h @@ -38,7 +38,7 @@ class BaseModel { BaseModel(std::shared_ptr& adapter, const ov::AnyMap& configuration = {}); virtual std::shared_ptr preprocess(const InputData& inputData, InferenceInput& input); - virtual std::unique_ptr postprocess(InferenceResult& infResult) = 0; + virtual std::unique_ptr postprocess(InferenceResult& infResult) = 0; void load(ov::Core& core, const std::string& device, size_t num_infer_requests = 1); @@ -49,7 +49,7 @@ class BaseModel { virtual void awaitAll(); virtual void awaitAny(); virtual void setCallback( - std::function, const ov::AnyMap& callback_args)> callback); + std::function, const ov::AnyMap& callback_args)> callback); std::shared_ptr getModel(); std::shared_ptr getInferenceAdapter(); @@ -67,8 +67,8 @@ class BaseModel { const std::vector& scale, const std::type_info& dtype = typeid(int)); virtual void inferAsync(const ImageInputData& inputData, const ov::AnyMap& callback_args = {}); - std::unique_ptr inferImage(const ImageInputData& inputData); - std::vector> inferBatchImage(const std::vector& inputData); + std::unique_ptr inferImage(const ImageInputData& inputData); + std::vector> inferBatchImage(const std::vector& inputData); protected: RESIZE_MODE selectResizeMode(const std::string& resize_type); @@ -104,5 +104,5 @@ class BaseModel { std::shared_ptr inferenceAdapter; std::map inputsLayouts; ov::Layout getInputLayout(const ov::Output& input); - std::function, const ov::AnyMap&)> lastCallback; + std::function, const ov::AnyMap&)> lastCallback; }; diff --git a/src/cpp/models/include/models/classification_model.h b/src/cpp/models/include/models/classification_model.h index 88ac03bc..529d5e13 100644 --- a/src/cpp/models/include/models/classification_model.h +++ b/src/cpp/models/include/models/classification_model.h @@ -99,10 +99,10 @@ class ClassificationModel : public BaseModel { const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; - virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch(const std::vector& inputImgs); + virtual std::unique_ptr infer(const ImageInputData& inputData); + virtual std::vector> inferBatch(const std::vector& inputImgs); static std::string ModelType; protected: @@ -119,8 +119,8 @@ class ClassificationModel : public BaseModel { void init_from_config(const ov::AnyMap& top_priority, const ov::AnyMap& mid_priority); void prepareInputsOutputs(std::shared_ptr& model) override; void updateModelInfo() override; - std::unique_ptr get_multilabel_predictions(InferenceResult& infResult, bool add_raw_scores); - std::unique_ptr get_multiclass_predictions(InferenceResult& infResult, bool add_raw_scores); - std::unique_ptr get_hierarchical_predictions(InferenceResult& infResult, bool add_raw_scores); + std::unique_ptr get_multilabel_predictions(InferenceResult& infResult, bool add_raw_scores); + std::unique_ptr get_multiclass_predictions(InferenceResult& infResult, bool add_raw_scores); + std::unique_ptr get_hierarchical_predictions(InferenceResult& infResult, bool add_raw_scores); ov::Tensor reorder_saliency_maps(const ov::Tensor&); }; diff --git a/src/cpp/models/include/models/detection_model.h b/src/cpp/models/include/models/detection_model.h index 16ba8cf8..9ec7ab8f 100644 --- a/src/cpp/models/include/models/detection_model.h +++ b/src/cpp/models/include/models/detection_model.h @@ -25,8 +25,8 @@ class DetectionModel : public BaseModel { const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); - virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch(const std::vector& inputImgs); + virtual std::unique_ptr infer(const ImageInputData& inputData); + virtual std::vector> inferBatch(const std::vector& inputImgs); protected: float confidence_threshold = 0.5f; diff --git a/src/cpp/models/include/models/detection_model_ssd.h b/src/cpp/models/include/models/detection_model_ssd.h index acb3060f..188301c5 100644 --- a/src/cpp/models/include/models/detection_model_ssd.h +++ b/src/cpp/models/include/models/detection_model_ssd.h @@ -25,12 +25,12 @@ class ModelSSD : public DetectionModel { public: using DetectionModel::DetectionModel; std::shared_ptr preprocess(const InputData& inputData, InferenceInput& input) override; - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; static std::string ModelType; protected: - std::unique_ptr postprocessSingleOutput(InferenceResult& infResult); - std::unique_ptr postprocessMultipleOutputs(InferenceResult& infResult); + std::unique_ptr postprocessSingleOutput(InferenceResult& infResult); + std::unique_ptr postprocessMultipleOutputs(InferenceResult& infResult); void prepareInputsOutputs(std::shared_ptr& model) override; void prepareSingleOutput(std::shared_ptr& model); void prepareMultipleOutputs(std::shared_ptr& model); diff --git a/src/cpp/models/include/models/detection_model_yolo.h b/src/cpp/models/include/models/detection_model_yolo.h index 56055588..491ee50b 100644 --- a/src/cpp/models/include/models/detection_model_yolo.h +++ b/src/cpp/models/include/models/detection_model_yolo.h @@ -16,9 +16,7 @@ #include "models/detection_model_ext.h" -struct DetectedObject; struct InferenceResult; -struct ResultBase; class ModelYolo : public DetectionModelExt { protected: @@ -46,7 +44,7 @@ class ModelYolo : public DetectionModelExt { ModelYolo(std::shared_ptr& model, const ov::AnyMap& configuration); ModelYolo(std::shared_ptr& adapter); - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; protected: void prepareInputsOutputs(std::shared_ptr& model) override; @@ -57,10 +55,10 @@ class ModelYolo : public DetectionModelExt { const unsigned long resized_im_w, const unsigned long original_im_h, const unsigned long original_im_w, - std::vector& objects); + std::vector& objects); static int calculateEntryIndex(int entriesNum, int lcoords, size_t lclasses, int location, int entry); - static double intersectionOverUnion(const DetectedObject& o1, const DetectedObject& o2); + static double intersectionOverUnion(const Box& o1, const Box& o2); std::map regions; float iou_threshold; @@ -82,7 +80,7 @@ class YOLOv5 : public DetectionModelExt { public: YOLOv5(std::shared_ptr& model, const ov::AnyMap& configuration); YOLOv5(std::shared_ptr& adapter); - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; static std::string ModelType; }; diff --git a/src/cpp/models/include/models/detection_model_yolov3_onnx.h b/src/cpp/models/include/models/detection_model_yolov3_onnx.h index 9dead24d..a11c900a 100644 --- a/src/cpp/models/include/models/detection_model_yolov3_onnx.h +++ b/src/cpp/models/include/models/detection_model_yolov3_onnx.h @@ -17,7 +17,7 @@ class ModelYoloV3ONNX : public DetectionModel { ModelYoloV3ONNX(std::shared_ptr& adapter); using DetectionModel::DetectionModel; - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; std::shared_ptr preprocess(const InputData& inputData, InferenceInput& input) override; protected: diff --git a/src/cpp/models/include/models/detection_model_yolox.h b/src/cpp/models/include/models/detection_model_yolox.h index bc747ee5..1849ba68 100644 --- a/src/cpp/models/include/models/detection_model_yolox.h +++ b/src/cpp/models/include/models/detection_model_yolox.h @@ -17,7 +17,7 @@ class ModelYoloX : public DetectionModelExt { ModelYoloX(std::shared_ptr& adapter); using DetectionModelExt::DetectionModelExt; - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; std::shared_ptr preprocess(const InputData& inputData, InferenceInput& input) override; static std::string ModelType; diff --git a/src/cpp/models/include/models/instance_segmentation.h b/src/cpp/models/include/models/instance_segmentation.h index c6cadce7..48b742dc 100644 --- a/src/cpp/models/include/models/instance_segmentation.h +++ b/src/cpp/models/include/models/instance_segmentation.h @@ -30,10 +30,10 @@ class MaskRCNNModel : public BaseModel { const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; - virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch( + virtual std::unique_ptr infer(const ImageInputData& inputData); + virtual std::vector> inferBatch( const std::vector& inputImgs); static std::string ModelType; bool postprocess_semantic_masks = true; @@ -49,4 +49,4 @@ class MaskRCNNModel : public BaseModel { float confidence_threshold = 0.5f; }; -cv::Mat segm_postprocess(const SegmentedObject& box, const cv::Mat& unpadded, int im_h, int im_w); +cv::Mat segm_postprocess(const Mask& box, const cv::Mat& unpadded, int im_h, int im_w); diff --git a/src/cpp/models/include/models/keypoint_detection.h b/src/cpp/models/include/models/keypoint_detection.h index 15d21cba..6fbea292 100644 --- a/src/cpp/models/include/models/keypoint_detection.h +++ b/src/cpp/models/include/models/keypoint_detection.h @@ -29,10 +29,10 @@ class KeypointDetectionModel : public BaseModel { const std::string& device = "AUTO"); static std::unique_ptr create_model(std::shared_ptr& adapter); - std::unique_ptr postprocess(InferenceResult& infResult) override; + std::unique_ptr postprocess(InferenceResult& infResult) override; - virtual std::unique_ptr infer(const ImageInputData& inputData); - virtual std::vector> inferBatch( + virtual std::unique_ptr infer(const ImageInputData& inputData); + virtual std::vector> inferBatch( const std::vector& inputImgs); static std::string ModelType; diff --git a/src/cpp/models/include/models/results.h b/src/cpp/models/include/models/results.h index 1a648723..fa93b9ee 100644 --- a/src/cpp/models/include/models/results.h +++ b/src/cpp/models/include/models/results.h @@ -16,69 +16,17 @@ #include "internal_model_data.h" struct MetaData; -struct ResultBase { - ResultBase(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) + +struct InferenceResult { + InferenceResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) : frameId(frameId), metaData(metaData) {} - virtual ~ResultBase() {} + std::shared_ptr internalModelData; + std::map outputsData; int64_t frameId; - std::shared_ptr metaData; - bool IsEmpty() { - return frameId < 0; - } - template - T& asRef() { - return dynamic_cast(*this); - } - - template - const T& asRef() const { - return dynamic_cast(*this); - } -}; - -struct AnomalyResult : public ResultBase { - AnomalyResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) - : ResultBase(frameId, metaData) {} - cv::Mat anomaly_map; - std::vector pred_boxes; - std::string pred_label; - cv::Mat pred_mask; - double pred_score; - - friend std::ostream& operator<<(std::ostream& os, const AnomalyResult& prediction) { - double min_anomaly_map, max_anomaly_map; - cv::minMaxLoc(prediction.anomaly_map, &min_anomaly_map, &max_anomaly_map); - double min_pred_mask, max_pred_mask; - cv::minMaxLoc(prediction.pred_mask, &min_pred_mask, &max_pred_mask); - os << "anomaly_map min:" << min_anomaly_map << " max:" << max_anomaly_map << ";"; - os << "pred_score:" << std::fixed << std::setprecision(1) << prediction.pred_score << ";"; - os << "pred_label:" << prediction.pred_label << ";"; - os << std::fixed << std::setprecision(0) << "pred_mask min:" << min_pred_mask << " max:" << max_pred_mask - << ";"; - - if (!prediction.pred_boxes.empty()) { - os << "pred_boxes:"; - for (const cv::Rect& box : prediction.pred_boxes) { - os << box << ","; - } - } - - return os; - } - explicit operator std::string() { - std::stringstream ss; - ss << *this; - return ss.str(); - } -}; - -struct InferenceResult : public ResultBase { - std::shared_ptr internalModelData; - std::map outputsData; /// Returns the first output tensor /// This function is a useful addition to direct access to outputs list as many models have only one output @@ -97,29 +45,20 @@ struct InferenceResult : public ResultBase { } }; -struct ClassificationResult : public ResultBase { - ClassificationResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) - : ResultBase(frameId, metaData) {} +struct DetectedKeypoints { + std::vector keypoints; + std::vector scores; - friend std::ostream& operator<<(std::ostream& os, const ClassificationResult& prediction) { - for (const ClassificationResult::Classification& classification : prediction.topLabels) { - os << classification << ", "; - } - try { - os << prediction.saliency_map.get_shape() << ", "; - } catch (ov::Exception&) { - os << "[0], "; - } - try { - os << prediction.feature_vector.get_shape() << ", "; - } catch (ov::Exception&) { - os << "[0], "; - } - try { - os << prediction.raw_scores.get_shape(); - } catch (ov::Exception&) { - os << "[0]"; + friend std::ostream& operator<<(std::ostream& os, const DetectedKeypoints& prediction) { + float kp_x_sum = 0.f; + for (const cv::Point2f& keypoint : prediction.keypoints) { + kp_x_sum += keypoint.x; } + float scores_sum = std::accumulate(prediction.scores.begin(), prediction.scores.end(), 0.f); + + os << "keypoints: (" << prediction.keypoints.size() << ", 2), keypoints_x_sum: "; + os << std::fixed << std::setprecision(3) << kp_x_sum << ", scores: (" << prediction.scores.size() << ",) " + << std::fixed << std::setprecision(3) << scores_sum; return os; } @@ -128,57 +67,86 @@ struct ClassificationResult : public ResultBase { ss << *this; return ss.str(); } +}; - struct Classification { - size_t id; - std::string label; - float score; +class Label { +public: + Label() {} + Label(int id, std::string name, float score): id(id), name(name), score(score) {} - Classification(size_t id, const std::string& label, float score) : id(id), label(label), score(score) {} + int id; + std::string name; + float score; - friend std::ostream& operator<<(std::ostream& os, const Classification& prediction) { - return os << prediction.id << " (" << prediction.label << "): " << std::fixed << std::setprecision(3) - << prediction.score; - } - }; + friend std::ostream& operator<< (std::ostream& os, const Label& label) { + return os << label.id << " (" << label.name << ")" << ": " << std::fixed << std::setprecision(3) << label.score; + } +}; + +class Mask { +public: + Mask(Label label, cv::Rect roi, cv::Mat mask): label(label), roi(roi), mask(mask) {} + + Label label; + cv::Rect roi; + cv::Mat mask; + + friend std::ostream& operator<< (std::ostream& os, const Mask& mask) { + + double min_mask, max_mask; + cv::minMaxLoc(mask.mask, &min_mask, &max_mask); + os << mask.label << mask.roi << " min:" << min_mask << " max:" << max_mask << ";"; + return os; + } - std::vector topLabels; - ov::Tensor saliency_map, feature_vector, - raw_scores; // Contains "raw_scores", "saliency_map" and "feature_vector" model outputs if such exist }; -struct DetectedObject : public cv::Rect2f { - size_t labelID; +struct Contour { std::string label; - float confidence; + float probability; + std::vector shape; - friend std::ostream& operator<<(std::ostream& os, const DetectedObject& detection) { - return os << int(detection.x) << ", " << int(detection.y) << ", " << int(detection.x + detection.width) << ", " - << int(detection.y + detection.height) << ", " << detection.labelID << " (" << detection.label - << "): " << std::fixed << std::setprecision(3) << detection.confidence; + friend std::ostream& operator<<(std::ostream& os, const Contour& contour) { + return os << contour.label << ": " << std::fixed << std::setprecision(3) << contour.probability << ", " + << contour.shape.size(); } }; -struct DetectionResult : public ResultBase { - DetectionResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) - : ResultBase(frameId, metaData) {} - std::vector objects; - ov::Tensor saliency_map, feature_vector; // Contan "saliency_map" and "feature_vector" model outputs if such exist - - friend std::ostream& operator<<(std::ostream& os, const DetectionResult& prediction) { - for (const DetectedObject& obj : prediction.objects) { - os << obj << "; "; - } - try { - os << prediction.saliency_map.get_shape() << "; "; - } catch (ov::Exception&) { - os << "[0]; "; +static inline std::vector getContours(const std::vector& segmentedObjects) { + std::vector combined_contours; + std::vector> contours; + for (const Mask& obj : segmentedObjects) { + cv::findContours(obj.mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_NONE); + // Assuming one contour output for findContours. Based on OTX this is a safe + // assumption + if (contours.size() != 1) { + throw std::runtime_error("findContours() must have returned only one contour"); } - try { - os << prediction.feature_vector.get_shape(); - } catch (ov::Exception&) { - os << "[0]"; + combined_contours.push_back({obj.label.name, obj.label.score, contours[0]}); + } + return combined_contours; +} + +class Box { +public: + Box(cv::Rect shape, std::vector