From 2394fff6bf832d6a1a4fc6485028f69cb7c915d3 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 17 Jan 2025 02:11:08 +0900 Subject: [PATCH 1/3] Preview of DetectionResult refactoring --- examples/cpp/CMakeLists.txt | 2 +- examples/cpp/asynchronous_api/main.cpp | 2 +- examples/cpp/synchronous_api/main.cpp | 9 ++++++-- src/cpp/models/include/models/results.h | 26 ++++++++++++++++++++-- src/cpp/models/src/detection_model_ssd.cpp | 24 ++++++++++++++++++++ 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 2fe5a7fd..b1ebab76 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -92,7 +92,7 @@ endmacro() find_package(OpenCV REQUIRED COMPONENTS imgcodecs) -set (ENABLE_PY_BINDINGS OFF) +set(ENABLE_PY_BINDINGS OFF) add_subdirectory(../../src/cpp ${Samples_BINARY_DIR}/src/cpp) add_example(NAME asynchronous_api SOURCES ./asynchronous_api/main.cpp DEPENDENCIES model_api) diff --git a/examples/cpp/asynchronous_api/main.cpp b/examples/cpp/asynchronous_api/main.cpp index acd361a2..105a5150 100644 --- a/examples/cpp/asynchronous_api/main.cpp +++ b/examples/cpp/asynchronous_api/main.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/examples/cpp/synchronous_api/main.cpp b/examples/cpp/synchronous_api/main.cpp index ec4ef869..edeb9b08 100644 --- a/examples/cpp/synchronous_api/main.cpp +++ b/examples/cpp/synchronous_api/main.cpp @@ -13,8 +13,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -41,6 +40,12 @@ int main(int argc, char* argv[]) try { << std::setw(4) << int(obj.x) << " | " << std::setw(4) << int(obj.y) << " | " << std::setw(4) << int(obj.x + obj.width) << " | " << std::setw(4) << int(obj.y + obj.height) << "\n"; } + + // could be improved with an iterator + for (size_t i = 0; i < result->size(); ++i) { + std::cout << result->getDetection(i) << "\n"; + } + } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; diff --git a/src/cpp/models/include/models/results.h b/src/cpp/models/include/models/results.h index 12673b3b..cad986ac 100644 --- a/src/cpp/models/include/models/results.h +++ b/src/cpp/models/include/models/results.h @@ -147,6 +147,9 @@ struct ClassificationResult : public ResultBase { }; struct DetectedObject : public cv::Rect2f { + DetectedObject() {} + DetectedObject(const cv::Rect2f& rect) : cv::Rect2f(rect) {} + size_t labelID; std::string label; float confidence; @@ -161,8 +164,27 @@ struct DetectedObject : public cv::Rect2f { struct DetectionResult : public ResultBase { DetectionResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) : ResultBase(frameId, metaData) {} - std::vector objects; - ov::Tensor saliency_map, feature_vector; // Contan "saliency_map" and "feature_vector" model outputs if such exist + + // memory to forward as np.array + cv::Mat_ scores; + cv::Mat_ bboxes; + cv::Mat_ labels; + std::vector label_names; + + std::vector objects; // to remove + ov::Tensor saliency_map, feature_vector; // Contain "saliency_map" and "feature_vector" model outputs if such exist + + size_t size() const { + return label_names.size(); + } + + DetectedObject getDetection(size_t idx) { + DetectedObject result(bboxes.at(idx)); + result.labelID = labels.at(idx); + result.label = label_names[idx]; + result.confidence = scores.at(idx); + return result; + } friend std::ostream& operator<<(std::ostream& os, const DetectionResult& prediction) { for (const DetectedObject& obj : prediction.objects) { diff --git a/src/cpp/models/src/detection_model_ssd.cpp b/src/cpp/models/src/detection_model_ssd.cpp index 20ed396a..911f5578 100644 --- a/src/cpp/models/src/detection_model_ssd.cpp +++ b/src/cpp/models/src/detection_model_ssd.cpp @@ -131,6 +131,7 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i } } + result->label_names.reserve(numAndStep.detectionsNum); for (size_t i = 0; i < numAndStep.detectionsNum; i++) { float image_id = detections[i * numAndStep.objectSize + 0]; if (image_id < 0) { @@ -165,6 +166,12 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i floatInputImgHeight) - desc.y; result->objects.push_back(desc); + + auto label_idx = static_cast(detections[i * numAndStep.objectSize + 1]); + result->labels.at(i) = label_idx; + result->scores.at(i) = confidence; + result->bboxes.at(i) = cv::Rect2f(desc.x, desc.y, desc.width, desc.height); + result->label_names.push_back(getLabelName(label_idx)); } } @@ -200,6 +207,18 @@ std::unique_ptr ModelSSD::postprocessMultipleOutputs(InferenceResult float widthScale = scores ? netInputWidth : 1.0f; float heightScale = scores ? netInputHeight : 1.0f; + size_t detections_num = 0; + for (size_t i = 0; i < numAndStep.detectionsNum; i++) { + float confidence = scores ? scores[i] : boxes[i * numAndStep.objectSize + 4]; + if (confidence > confidence_threshold) { + ++detections_num; + } + } + result->label_names.reserve(detections_num); + result->labels = cv::Mat(1, detections_num, CV_32S); + result->scores = cv::Mat(1, detections_num, CV_32F); + result->bboxes = cv::Mat(1, detections_num, CV_32FC4); + for (size_t i = 0; i < numAndStep.detectionsNum; i++) { float confidence = scores ? scores[i] : boxes[i * numAndStep.objectSize + 4]; @@ -229,6 +248,11 @@ std::unique_ptr ModelSSD::postprocessMultipleOutputs(InferenceResult if (desc.width * desc.height >= box_area_threshold) { result->objects.push_back(desc); } + auto label_idx = static_cast(labels[i]); + result->labels.at(i) = label_idx; + result->scores.at(i) = confidence; + result->bboxes.at(i) = cv::Rect2f(desc.x, desc.y, desc.width, desc.height); + result->label_names.push_back(getLabelName(label_idx)); } } From 4c0b786e552a5b62e71399eca6b4b4aadf7fa947 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 17 Jan 2025 23:40:26 +0900 Subject: [PATCH 2/3] Fix labeler --- .github/labeler.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 32d1c511..278856a6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -3,12 +3,12 @@ cpp: - changed-files: - any-glob-to-any-file: - - model_api/cpp/** + - src/cpp/** python: - changed-files: - any-glob-to-any-file: - - model_api/python/** + - src/python/** tests: - changed-files: From bc3536050e292219dc1bfbcd92831689915ba0f2 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Sat, 18 Jan 2025 00:12:54 +0900 Subject: [PATCH 3/3] Add an iterator through detections --- examples/cpp/synchronous_api/main.cpp | 8 +---- src/cpp/models/include/models/results.h | 35 ++++++++++++++++--- src/cpp/models/src/detection_model_ssd.cpp | 40 +++++++++++++--------- 3 files changed, 54 insertions(+), 29 deletions(-) diff --git a/examples/cpp/synchronous_api/main.cpp b/examples/cpp/synchronous_api/main.cpp index edeb9b08..7745fd36 100644 --- a/examples/cpp/synchronous_api/main.cpp +++ b/examples/cpp/synchronous_api/main.cpp @@ -35,17 +35,11 @@ int main(int argc, char* argv[]) try { auto result = model->infer(image); // Process detections - for (auto& obj : result->objects) { + for (auto obj : *result) { std::cout << " " << std::left << std::setw(9) << obj.label << " | " << std::setw(10) << obj.confidence << " | " << std::setw(4) << int(obj.x) << " | " << std::setw(4) << int(obj.y) << " | " << std::setw(4) << int(obj.x + obj.width) << " | " << std::setw(4) << int(obj.y + obj.height) << "\n"; } - - // could be improved with an iterator - for (size_t i = 0; i < result->size(); ++i) { - std::cout << result->getDetection(i) << "\n"; - } - } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; diff --git a/src/cpp/models/include/models/results.h b/src/cpp/models/include/models/results.h index cad986ac..b211e7cf 100644 --- a/src/cpp/models/include/models/results.h +++ b/src/cpp/models/include/models/results.h @@ -165,20 +165,45 @@ struct DetectionResult : public ResultBase { DetectionResult(int64_t frameId = -1, const std::shared_ptr& metaData = nullptr) : ResultBase(frameId, metaData) {} - // memory to forward as np.array cv::Mat_ scores; cv::Mat_ bboxes; cv::Mat_ labels; std::vector label_names; - - std::vector objects; // to remove ov::Tensor saliency_map, feature_vector; // Contain "saliency_map" and "feature_vector" model outputs if such exist + struct DetectionIterator { + const DetectionResult& result; + size_t index; + + DetectionIterator(const DetectionResult& result, size_t index) : result(result), index(index) {} + + bool operator!=(const DetectionIterator& other) const { + return index != other.index; + } + + DetectionIterator& operator++() { + ++index; + return *this; + } + + DetectedObject operator*() const { + return result.getObject(index); + } + }; + + DetectionIterator begin() const { + return DetectionIterator(*this, 0); + } + + DetectionIterator end() const { + return DetectionIterator(*this, size()); + } + size_t size() const { return label_names.size(); } - DetectedObject getDetection(size_t idx) { + DetectedObject getObject(size_t idx) const { DetectedObject result(bboxes.at(idx)); result.labelID = labels.at(idx); result.label = label_names[idx]; @@ -187,7 +212,7 @@ struct DetectionResult : public ResultBase { } friend std::ostream& operator<<(std::ostream& os, const DetectionResult& prediction) { - for (const DetectedObject& obj : prediction.objects) { + for (const DetectedObject obj : prediction) { os << obj << "; "; } try { diff --git a/src/cpp/models/src/detection_model_ssd.cpp b/src/cpp/models/src/detection_model_ssd.cpp index 911f5578..74ab1c13 100644 --- a/src/cpp/models/src/detection_model_ssd.cpp +++ b/src/cpp/models/src/detection_model_ssd.cpp @@ -131,7 +131,18 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i } } - result->label_names.reserve(numAndStep.detectionsNum); + size_t detections_num = 0; + for (size_t i = 0; i < numAndStep.detectionsNum; i++) { + float confidence = detections[i * numAndStep.objectSize + 2]; + if (confidence > confidence_threshold) { + ++detections_num; + } + } + result->label_names.reserve(detections_num); + result->labels = cv::Mat(1, detections_num, CV_32S); + result->scores = cv::Mat(1, detections_num, CV_32F); + result->bboxes = cv::Mat(1, detections_num, CV_32FC4); + for (size_t i = 0; i < numAndStep.detectionsNum; i++) { float image_id = detections[i * numAndStep.objectSize + 0]; if (image_id < 0) { @@ -142,11 +153,7 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i /** Filtering out objects with confidence < confidence_threshold probability **/ if (confidence > confidence_threshold) { - DetectedObject desc; - - desc.confidence = confidence; - desc.labelID = static_cast(detections[i * numAndStep.objectSize + 1]); - desc.label = getLabelName(desc.labelID); + cv::Rect2f desc; desc.x = clamp(round((detections[i * numAndStep.objectSize + 3] * netInputWidth - padLeft) * invertedScaleX), 0.f, @@ -165,12 +172,12 @@ std::unique_ptr ModelSSD::postprocessSingleOutput(InferenceResult& i 0.f, floatInputImgHeight) - desc.y; - result->objects.push_back(desc); + size_t det_idx = result->label_names.size(); auto label_idx = static_cast(detections[i * numAndStep.objectSize + 1]); - result->labels.at(i) = label_idx; - result->scores.at(i) = confidence; - result->bboxes.at(i) = cv::Rect2f(desc.x, desc.y, desc.width, desc.height); + result->labels.at(det_idx) = label_idx; + result->scores.at(det_idx) = confidence; + result->bboxes.at(det_idx) = desc; result->label_names.push_back(getLabelName(label_idx)); } } @@ -224,11 +231,8 @@ std::unique_ptr ModelSSD::postprocessMultipleOutputs(InferenceResult /** Filtering out objects with confidence < confidence_threshold probability **/ if (confidence > confidence_threshold) { - DetectedObject desc; + cv::Rect2f desc; - desc.confidence = confidence; - desc.labelID = labels[i]; - desc.label = getLabelName(desc.labelID); desc.x = clamp_and_round((boxes[i * numAndStep.objectSize] * widthScale - padLeft) * invertedScaleX, 0.f, floatInputImgWidth); @@ -248,10 +252,12 @@ std::unique_ptr ModelSSD::postprocessMultipleOutputs(InferenceResult if (desc.width * desc.height >= box_area_threshold) { result->objects.push_back(desc); } + + size_t det_idx = result->label_names.size(); auto label_idx = static_cast(labels[i]); - result->labels.at(i) = label_idx; - result->scores.at(i) = confidence; - result->bboxes.at(i) = cv::Rect2f(desc.x, desc.y, desc.width, desc.height); + result->labels.at(det_idx) = label_idx; + result->scores.at(det_idx) = confidence; + result->bboxes.at(det_idx) = desc; result->label_names.push_back(getLabelName(label_idx)); } }