Skip to content

Commit 46f4eb7

Browse files
RHeckerIntelRHeckerIntel
committed
Implement anomaly detection. WIP
Havent verified it works correctly. Serialization probably broken.
1 parent 25e44e7 commit 46f4eb7

File tree

5 files changed

+308
-0
lines changed

5 files changed

+308
-0
lines changed

‎src/cpp/include/tasks/anomaly.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright (C) 2020-2025 Intel Corporation
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
#pragma once
7+
8+
#include <opencv2/opencv.hpp>
9+
#include <openvino/openvino.hpp>
10+
11+
#include "adapters/inference_adapter.h"
12+
#include "tasks/results.h"
13+
#include "utils/config.h"
14+
#include "utils/vision_pipeline.h"
15+
16+
17+
class Anomaly {
18+
public:
19+
std::shared_ptr<InferenceAdapter> adapter;
20+
VisionPipeline<AnomalyResult> pipeline;
21+
22+
Anomaly(std::shared_ptr<InferenceAdapter> adapter, cv::Size input_shape)
23+
: adapter(adapter),
24+
input_shape(input_shape) {
25+
pipeline = VisionPipeline<AnomalyResult>(
26+
adapter,
27+
[&](cv::Mat image) {
28+
return preprocess(image);
29+
},
30+
[&](InferenceResult result) {
31+
return postprocess(result);
32+
});
33+
34+
auto config = adapter->getModelConfig();
35+
image_threshold = utils::get_from_any_maps("image_threshold", config, {}, image_threshold);
36+
pixel_threshold = utils::get_from_any_maps("pixel_threshold", config, {}, pixel_threshold);
37+
normalization_scale = utils::get_from_any_maps("pixel_threshold", config, {}, normalization_scale);
38+
task = utils::get_from_any_maps("pixel_threshold", config, {}, task);
39+
labels = utils::get_from_any_maps("labels", config, {}, labels);
40+
41+
//labels = utils::get_from_any_maps("labels", config, {}, labels);
42+
//confidence_threshold = utils::get_from_any_maps("confidence_threshold", config, {}, confidence_threshold);
43+
}
44+
45+
static cv::Size serialize(std::shared_ptr<ov::Model>& ov_model);
46+
static Anomaly load(const std::string& model_path);
47+
48+
AnomalyResult infer(cv::Mat image);
49+
std::vector<AnomalyResult> inferBatch(std::vector<cv::Mat> image);
50+
51+
std::map<std::string, ov::Tensor> preprocess(cv::Mat);
52+
AnomalyResult postprocess(InferenceResult& infResult);
53+
54+
private:
55+
cv::Mat normalize(cv::Mat& tensor, float threshold);
56+
double normalize(double& tensor, float threshold);
57+
std::vector<cv::Rect> getBoxes(cv::Mat& mask);
58+
59+
60+
private:
61+
cv::Size input_shape;
62+
std::vector<std::string> labels;
63+
64+
float image_threshold = 0.5f;
65+
float pixel_threshold = 0.5f;
66+
float normalization_scale = 1.0f;
67+
std::string task = "segmentation";
68+
};

‎src/cpp/include/tasks/results.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,3 +146,37 @@ struct InstanceSegmentationResult {
146146
std::vector<cv::Mat_<std::uint8_t>> saliency_map;
147147
ov::Tensor feature_vector;
148148
};
149+
150+
struct AnomalyResult {
151+
cv::Mat anomaly_map;
152+
std::vector<cv::Rect> pred_boxes;
153+
std::string pred_label;
154+
cv::Mat pred_mask;
155+
double pred_score;
156+
157+
friend std::ostream& operator<<(std::ostream& os, const AnomalyResult& prediction) {
158+
double min_anomaly_map, max_anomaly_map;
159+
cv::minMaxLoc(prediction.anomaly_map, &min_anomaly_map, &max_anomaly_map);
160+
double min_pred_mask, max_pred_mask;
161+
cv::minMaxLoc(prediction.pred_mask, &min_pred_mask, &max_pred_mask);
162+
os << "anomaly_map min:" << min_anomaly_map << " max:" << max_anomaly_map << ";";
163+
os << "pred_score:" << std::fixed << std::setprecision(1) << prediction.pred_score << ";";
164+
os << "pred_label:" << prediction.pred_label << ";";
165+
os << std::fixed << std::setprecision(0) << "pred_mask min:" << min_pred_mask << " max:" << max_pred_mask
166+
<< ";";
167+
168+
if (!prediction.pred_boxes.empty()) {
169+
os << "pred_boxes:";
170+
for (const cv::Rect& box : prediction.pred_boxes) {
171+
os << box << ",";
172+
}
173+
}
174+
175+
return os;
176+
}
177+
explicit operator std::string() {
178+
std::stringstream ss;
179+
ss << *this;
180+
return ss.str();
181+
}
182+
};

‎src/cpp/src/tasks/anomaly.cpp‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#include "tasks/anomaly.h"
2+
#include "adapters/openvino_adapter.h"
3+
#include "utils/tensor.h"
4+
#include "utils/preprocessing.h"
5+
6+
7+
cv::Size Anomaly::serialize(std::shared_ptr<ov::Model>& ov_model) {
8+
auto input = ov_model->inputs().front();
9+
10+
auto layout = ov::layout::get_layout(input);
11+
if (layout.empty()) {
12+
layout = utils::getLayoutFromShape(input.get_partial_shape());
13+
}
14+
15+
const ov::Shape& shape = input.get_partial_shape().get_max_shape();
16+
17+
auto interpolation_mode = cv::INTER_LINEAR;
18+
utils::RESIZE_MODE resize_mode = utils::RESIZE_FILL;
19+
uint8_t pad_value = 0;
20+
bool reverse_input_channels = false;
21+
22+
23+
std::vector<float> scale_values;
24+
std::vector<float> mean_values;
25+
if (ov_model->has_rt_info("model_info")) {
26+
auto config = ov_model->get_rt_info<ov::AnyMap>("model_info");
27+
scale_values = utils::get_from_any_maps("scale_values", config, ov::AnyMap{}, scale_values);
28+
mean_values = utils::get_from_any_maps("mean_values", config, ov::AnyMap{}, mean_values);
29+
}
30+
31+
auto input_shape = ov::Shape{shape[ov::layout::width_idx(layout)], shape[ov::layout::height_idx(layout)]};
32+
33+
ov_model = utils::embedProcessing(
34+
ov_model,
35+
input.get_any_name(),
36+
layout,
37+
resize_mode,
38+
interpolation_mode,
39+
input_shape,
40+
pad_value,
41+
reverse_input_channels,
42+
mean_values,
43+
scale_values);
44+
45+
return cv::Size(input_shape[0], input_shape[1]);
46+
}
47+
48+
Anomaly Anomaly::load(const std::string& model_path) {
49+
auto core = ov::Core();
50+
std::shared_ptr<ov::Model> model = core.read_model(model_path);
51+
52+
if (model->has_rt_info("model_info", "model_type")) {
53+
std::cout << "has model type in info: " << model->get_rt_info<std::string>("model_info", "model_type")
54+
<< std::endl;
55+
} else {
56+
throw std::runtime_error("Incorrect or unsupported model_type");
57+
}
58+
59+
cv::Size origin_input_shape;
60+
if (utils::model_has_embedded_processing(model)) {
61+
std::cout << "model already was serialized" << std::endl;
62+
origin_input_shape = utils::get_input_shape_from_model_info(model);
63+
} else {
64+
origin_input_shape = serialize(model);
65+
}
66+
auto adapter = std::make_shared<OpenVINOInferenceAdapter>();
67+
adapter->loadModel(model, core, "AUTO");
68+
return Anomaly(adapter, origin_input_shape);
69+
}
70+
71+
AnomalyResult Anomaly::infer(cv::Mat image) {
72+
return pipeline.infer(image);
73+
}
74+
75+
std::vector<AnomalyResult> Anomaly::inferBatch(std::vector<cv::Mat> images) {
76+
return pipeline.inferBatch(images);
77+
}
78+
79+
std::map<std::string, ov::Tensor> Anomaly::preprocess(cv::Mat image) {
80+
std::map<std::string, ov::Tensor> input = {};
81+
input.emplace(adapter->getInputNames()[0], utils::wrapMat2Tensor(image));
82+
return input;
83+
}
84+
85+
AnomalyResult Anomaly::postprocess(InferenceResult& infResult) {
86+
auto tensorName = adapter->getOutputNames().front();
87+
ov::Tensor predictions = infResult.data[tensorName];
88+
const auto& inputImgSize = infResult.inputImageSize;
89+
90+
double pred_score;
91+
std::string pred_label;
92+
cv::Mat anomaly_map;
93+
cv::Mat pred_mask;
94+
std::vector<cv::Rect> pred_boxes;
95+
if (predictions.get_shape().size() == 1) {
96+
pred_score = predictions.data<float>()[0];
97+
} else {
98+
const ov::Layout& layout = utils::getLayoutFromShape(predictions.get_shape());
99+
const ov::Shape& predictionsShape = predictions.get_shape();
100+
anomaly_map = cv::Mat(static_cast<int>(predictionsShape[ov::layout::height_idx(layout)]),
101+
static_cast<int>(predictionsShape[ov::layout::width_idx(layout)]),
102+
CV_32FC1,
103+
predictions.data<float>());
104+
// find the max predicted score
105+
cv::minMaxLoc(anomaly_map, NULL, &pred_score);
106+
}
107+
pred_label = labels[pred_score > image_threshold ? 1 : 0];
108+
109+
pred_mask = anomaly_map >= pixel_threshold;
110+
pred_mask.convertTo(pred_mask, CV_8UC1, 1 / 255.);
111+
cv::resize(pred_mask, pred_mask, cv::Size{inputImgSize.width, inputImgSize.height});
112+
anomaly_map = normalize(anomaly_map, pixel_threshold);
113+
anomaly_map.convertTo(anomaly_map, CV_8UC1, 255);
114+
115+
pred_score = normalize(pred_score, image_threshold);
116+
if (pred_label == labels[0]) { // normal label
117+
pred_score = 1 - pred_score; // Score of normal is 1 - score of anomaly
118+
}
119+
120+
if (!anomaly_map.empty()) {
121+
cv::resize(anomaly_map, anomaly_map, cv::Size{inputImgSize.width, inputImgSize.height});
122+
}
123+
124+
if (task == "detection") {
125+
pred_boxes = getBoxes(pred_mask);
126+
}
127+
128+
AnomalyResult result;
129+
result.anomaly_map = std::move(anomaly_map);
130+
result.pred_score = pred_score;
131+
result.pred_label = std::move(pred_label);
132+
result.pred_mask = std::move(pred_mask);
133+
result.pred_boxes = std::move(pred_boxes);
134+
return result;
135+
}
136+
137+
cv::Mat Anomaly::normalize(cv::Mat& tensor, float threshold) {
138+
cv::Mat normalized = ((tensor - threshold) / normalization_scale) + 0.5f;
139+
normalized = cv::min(cv::max(normalized, 0.f), 1.f);
140+
return normalized;
141+
}
142+
143+
double Anomaly::normalize(double& value, float threshold) {
144+
double normalized = ((value - threshold) / normalization_scale) + 0.5f;
145+
return std::min(std::max(normalized, 0.), 1.);
146+
}
147+
148+
std::vector<cv::Rect> Anomaly::getBoxes(cv::Mat& mask) {
149+
std::vector<cv::Rect> boxes;
150+
std::vector<std::vector<cv::Point>> contours;
151+
cv::findContours(mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
152+
for (auto& contour : contours) {
153+
std::vector<int> box;
154+
cv::Rect rect = cv::boundingRect(contour);
155+
boxes.push_back(rect);
156+
}
157+
return boxes;
158+
}

‎tests/cpp/test_accuracy.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <thread>
88

99
#include "matchers.h"
10+
#include "tasks/anomaly.h"
1011
#include "tasks/detection.h"
1112
#include "tasks/instance_segmentation.h"
1213
#include "tasks/semantic_segmentation.h"
@@ -102,6 +103,8 @@ TEST_P(ModelParameterizedTest, AccuracyTest) {
102103
// auto model = SemanticSegmentation::load(model_path);
103104
} else if (data.type == "MaskRCNNModel") {
104105
GTEST_SKIP();
106+
} else if (data.type == "AnomalyDetection") {
107+
GTEST_SKIP();
105108
} else {
106109
FAIL() << "No implementation for model type " << data.type;
107110
}
@@ -144,6 +147,16 @@ TEST_P(ModelParameterizedTest, SerializedAccuracyTest) {
144147

145148
EXPECT_EQ(format_test_output_to_string(model, result), test_data.reference[0]);
146149
}
150+
} else if (data.type == "AnomalyDetection") {
151+
auto model = Anomaly::load(model_path);
152+
153+
for (auto& test_data : data.test_data) {
154+
std::string image_path = DATA_DIR + '/' + test_data.image;
155+
cv::Mat image = cv::imread(image_path);
156+
auto result = model.infer(image);
157+
158+
EXPECT_EQ(std::string{result}, test_data.reference[0]);
159+
}
147160
} else {
148161
FAIL() << "No implementation for model type " << data.type;
149162
}
@@ -190,6 +203,17 @@ TEST_P(ModelParameterizedTest, AccuracyTestBatch) {
190203
ASSERT_EQ(result.size(), 1);
191204
EXPECT_EQ(format_test_output_to_string(model, result[0]), test_data.reference[0]);
192205
}
206+
} else if (data.type == "AnomalyDetection") {
207+
auto model = Anomaly::load(model_path);
208+
209+
for (auto& test_data : data.test_data) {
210+
std::string image_path = DATA_DIR + '/' + test_data.image;
211+
cv::Mat image = cv::imread(image_path);
212+
auto result = model.inferBatch({image});
213+
214+
ASSERT_EQ(result.size(), 1);
215+
EXPECT_EQ(std::string{result[0]}, test_data.reference[0]);
216+
}
193217
} else {
194218
FAIL() << "No implementation for model type " << data.type;
195219
}

‎tests/python/accuracy/public_scope.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,5 +96,29 @@
9696
]
9797
}
9898
]
99+
},
100+
{
101+
"name": "otx_models/anomaly_padim_bottle_mvtec.xml",
102+
"type": "AnomalyDetection",
103+
"test_data": [
104+
{
105+
"image": "coco128/images/train2017/000000000074.jpg",
106+
"reference": [
107+
"anomaly_map min:151 max:255;pred_score:1.0;pred_label:Anomaly;pred_mask min:1 max:1;"
108+
]
109+
}
110+
]
111+
},
112+
{
113+
"name": "otx_models/anomaly_stfpm_bottle_mvtec.xml",
114+
"type": "AnomalyDetection",
115+
"test_data": [
116+
{
117+
"image": "coco128/images/train2017/000000000074.jpg",
118+
"reference": [
119+
"anomaly_map min:124 max:225;pred_score:0.9;pred_label:Anomaly;pred_mask min:0 max:1;"
120+
]
121+
}
122+
]
99123
}
100124
]

0 commit comments

Comments
 (0)