Skip to content

Commit 25927d9

Browse files
RHeckerIntelRHeckerIntel
committed
Adds test_model_config tests partially back
Missing serialization. Need to allow for that again.
1 parent 8168c27 commit 25927d9

File tree

8 files changed

+339
-3
lines changed

8 files changed

+339
-3
lines changed

‎src/cpp/include/tasks/detection.h‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
class DetectionModel {
1919
public:
2020
std::unique_ptr<Pipeline<DetectionResult>> pipeline;
21+
std::unique_ptr<SSD> algorithm;
2122

2223
DetectionModel(std::unique_ptr<SSD> algorithm, const ov::AnyMap& user_config) : algorithm(std::move(algorithm)) {
2324
auto config = this->algorithm->adapter->getModelConfig();
@@ -71,7 +72,4 @@ class DetectionModel {
7172

7273
DetectionResult infer(cv::Mat image);
7374
std::vector<DetectionResult> inferBatch(std::vector<cv::Mat> image);
74-
75-
private:
76-
std::unique_ptr<SSD> algorithm;
7775
};

‎src/cpp/src/tasks/anomaly.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ void Anomaly::serialize(std::shared_ptr<ov::Model>& ov_model) {
4747
mean_values,
4848
scale_values);
4949

50+
ov_model->set_rt_info(true, "model_info", "embedded_processing");
5051
ov_model->set_rt_info(input_shape[0], "model_info", "orig_width");
5152
ov_model->set_rt_info(input_shape[1], "model_info", "orig_height");
5253
}

‎src/cpp/src/tasks/classification.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ void Classification::serialize(std::shared_ptr<ov::Model>& ov_model) {
176176
addOrFindSoftmaxAndTopkOutputs(ov_model, topk, output_raw_scores);
177177
}
178178

179+
ov_model->set_rt_info(true, "model_info", "embedded_processing");
179180
ov_model->set_rt_info(input_shape[0], "model_info", "orig_width");
180181
ov_model->set_rt_info(input_shape[1], "model_info", "orig_height");
181182
}

‎src/cpp/src/tasks/detection/ssd.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ void SSD::serialize(std::shared_ptr<ov::Model>& ov_model) {
115115
// prepareMultipleOutputs(ov_model); //This does nothing from what I can see.
116116
}
117117

118+
ov_model->set_rt_info(true, "model_info", "embedded_processing");
118119
ov_model->set_rt_info(input_shape[0], "model_info", "orig_width");
119120
ov_model->set_rt_info(input_shape[1], "model_info", "orig_height");
120121
}

‎src/cpp/src/tasks/instance_segmentation.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ void InstanceSegmentation::serialize(std::shared_ptr<ov::Model>& ov_model) {
186186
saliency_map_name + ", " + feature_vector_name + " and 3 or 4 other outputs");
187187
}
188188

189+
ov_model->set_rt_info(true, "model_info", "embedded_processing");
189190
ov_model->set_rt_info(input_shape.width, "model_info", "orig_width");
190191
ov_model->set_rt_info(input_shape.height, "model_info", "orig_height");
191192
}

‎src/cpp/src/tasks/semantic_segmentation.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ void SemanticSegmentation::serialize(std::shared_ptr<ov::Model>& ov_model) {
114114
ov_model = ppp.build();
115115

116116
cv::Size input_shape(shape[ov::layout::width_idx(layout)], shape[ov::layout::height_idx(layout)]);
117+
ov_model->set_rt_info(true, "model_info", "embedded_processing");
117118
ov_model->set_rt_info(input_shape.width, "model_info", "orig_width");
118119
ov_model->set_rt_info(input_shape.height, "model_info", "orig_height");
119120
}

‎tests/cpp/precommit/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,6 @@ find_package(OpenCV REQUIRED COMPONENTS imgcodecs)
2020

2121
add_executable(test_sanity test_sanity.cpp)
2222
target_link_libraries(test_sanity gtest_main nlohmann_json::nlohmann_json model_api ${OpenCV_LIBRARIES})
23+
24+
add_executable(test_model_config test_model_config.cpp)
25+
target_link_libraries(test_model_config gtest_main nlohmann_json::nlohmann_json model_api ${OpenCV_LIBRARIES})

‎tests/cpp/precommit/test_model_config.cpp‎

Lines changed: 330 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,330 @@
1+
/*
2+
* Copyright (C) 2020-2024 Intel Corporation
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
#include <adapters/openvino_adapter.h>
6+
#include <gtest/gtest.h>
7+
#include <tasks/classification.h>
8+
#include <tasks/detection.h>
9+
#include <tasks/results.h>
10+
#include <stddef.h>
11+
12+
#include <cstdint>
13+
#include <cstdio>
14+
#include <exception>
15+
#include <fstream>
16+
#include <iomanip>
17+
#include <iostream>
18+
#include <nlohmann/json.hpp>
19+
#include <opencv2/core.hpp>
20+
#include <stdexcept>
21+
#include <string>
22+
#include "utils/config.h"
23+
24+
using json = nlohmann::json;
25+
26+
std::string DATA_DIR = "../data";
27+
std::string MODEL_PATH_TEMPLATE = "otx_models/%s.xml";
28+
std::string IMAGE_PATH = "coco128/images/train2017/000000000074.jpg";
29+
30+
std::string TMP_MODEL_FILE = "tmp_model.xml";
31+
32+
struct ModelData {
33+
std::string name;
34+
ModelData(const std::string& name) : name(name) {}
35+
};
36+
37+
class MockAdapter : public OpenVINOInferenceAdapter {
38+
public:
39+
MockAdapter(const std::string& modelPath) : OpenVINOInferenceAdapter() {
40+
loadModel(modelPath, "CPU");
41+
}
42+
};
43+
44+
class ClassificationModelParameterizedTest : public testing::TestWithParam<ModelData> {};
45+
46+
class SSDModelParameterizedTest : public testing::TestWithParam<ModelData> {};
47+
48+
class ClassificationModelParameterizedTestSaveLoad : public testing::TestWithParam<ModelData> {
49+
protected:
50+
void TearDown() override {
51+
auto fileName = TMP_MODEL_FILE;
52+
std::remove(fileName.c_str());
53+
std::remove(fileName.replace(fileName.end() - 4, fileName.end(), ".bin").c_str());
54+
}
55+
};
56+
57+
class DetectionModelParameterizedTestSaveLoad : public ClassificationModelParameterizedTestSaveLoad {};
58+
59+
template <typename... Args>
60+
std::string string_format(const std::string& fmt, Args... args) {
61+
size_t size = snprintf(nullptr, 0, fmt.c_str(), args...);
62+
std::string buf;
63+
buf.reserve(size + 1);
64+
buf.resize(size);
65+
snprintf(&buf[0], size + 1, fmt.c_str(), args...);
66+
return buf;
67+
}
68+
69+
TEST_P(ClassificationModelParameterizedTest, TestClassificationDefaultConfig) {
70+
auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
71+
bool preload = true;
72+
auto model = Classification::create_model(DATA_DIR + "/" + model_path, {}, preload, "CPU");
73+
74+
auto config = model.adapter->getModelConfig();
75+
76+
std::string model_type;
77+
model_type = utils::get_from_any_maps("model_type", config, {}, model_type);
78+
EXPECT_EQ(model_type, "Classification");
79+
80+
bool embedded_processing;
81+
embedded_processing = utils::get_from_any_maps("embedded_processing", config, {}, embedded_processing);
82+
EXPECT_TRUE(embedded_processing);
83+
}
84+
85+
TEST_P(ClassificationModelParameterizedTest, TestClassificationCustomConfig) {
86+
GTEST_SKIP() << "Classification config tests fail on CI";
87+
auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
88+
std::vector<std::string> mock_labels;
89+
size_t num_classes = 1000;
90+
for (size_t i = 0; i < num_classes; i++) {
91+
mock_labels.push_back(std::to_string(i));
92+
}
93+
ov::AnyMap configuration = {{"layout", "data:HWC"}, {"resize_type", "fit_to_window"}, {"labels", mock_labels}};
94+
bool preload = true;
95+
auto model = Classification::create_model(DATA_DIR + "/" + model_path, configuration, preload, "CPU");
96+
97+
auto config = model.adapter->getModelConfig();
98+
std::string layout;
99+
layout = utils::get_from_any_maps("layout", config, {}, layout);
100+
EXPECT_EQ(layout, configuration.at("layout").as<std::string>());
101+
102+
std::string resize_type;
103+
resize_type = utils::get_from_any_maps("resize_type", config, {}, resize_type);
104+
EXPECT_EQ(resize_type, configuration.at("resize_type").as<std::string>());
105+
106+
std::vector<std::string> labels;
107+
labels = utils::get_from_any_maps("labels", config, {}, labels);
108+
for (size_t i = 0; i < num_classes; i++) {
109+
EXPECT_EQ(labels[i], mock_labels[i]);
110+
}
111+
}
112+
113+
//TEST_P(ClassificationModelParameterizedTestSaveLoad, TestClassificationCorrectnessAfterSaveLoad) {
114+
// cv::Mat image = cv::imread(DATA_DIR + "/" + IMAGE_PATH);
115+
// if (!image.data) {
116+
// throw std::runtime_error{"Failed to read the image"};
117+
// }
118+
//
119+
// auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
120+
// std::cout << model_path << "\n";
121+
// bool preload = true;
122+
// auto model = Classification::create_model(DATA_DIR + "/" + model_path, {}, preload, "CPU");
123+
//
124+
// auto ov_model = model->getModel();
125+
// ov::serialize(ov_model, TMP_MODEL_FILE);
126+
//
127+
// auto result = model->infer(image)->topLabels;
128+
//
129+
// auto model_restored = ClassificationModel::create_model(TMP_MODEL_FILE, {}, preload, "CPU");
130+
// auto result_data = model_restored->infer(image);
131+
// auto result_restored = result_data->topLabels;
132+
//
133+
// EXPECT_EQ(result_restored[0].id, result[0].id);
134+
// EXPECT_EQ(result_restored[0].score, result[0].score);
135+
//}
136+
//
137+
//TEST_P(ClassificationModelParameterizedTestSaveLoad, TestClassificationCorrectnessAfterSaveLoadWithAdapter) {
138+
// cv::Mat image = cv::imread(DATA_DIR + "/" + IMAGE_PATH);
139+
// if (!image.data) {
140+
// throw std::runtime_error{"Failed to read the image"};
141+
// }
142+
//
143+
// auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
144+
// bool preload = true;
145+
// auto model = ClassificationModel::create_model(DATA_DIR + "/" + model_path, {}, preload, "CPU");
146+
// auto ov_model = model->getModel();
147+
// ov::serialize(ov_model, TMP_MODEL_FILE);
148+
// auto result = model->infer(image)->topLabels;
149+
//
150+
// std::shared_ptr<InferenceAdapter> adapter = std::make_shared<MockAdapter>(TMP_MODEL_FILE);
151+
// auto model_restored = ClassificationModel::create_model(adapter);
152+
// auto result_data = model_restored->infer(image);
153+
// auto result_restored = result_data->topLabels;
154+
//
155+
// EXPECT_EQ(result_restored[0].id, result[0].id);
156+
// EXPECT_EQ(result_restored[0].score, result[0].score);
157+
//}
158+
159+
TEST_P(SSDModelParameterizedTest, TestDetectionDefaultConfig) {
160+
auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
161+
bool preload = true;
162+
auto model = DetectionModel::create_model(DATA_DIR + "/" + model_path, {}, preload, "CPU");
163+
164+
auto config = model.algorithm->adapter->getModelConfig();
165+
166+
std::string model_type;
167+
model_type = utils::get_from_any_maps("model_type", config, {}, model_type);
168+
EXPECT_EQ(model_type, "ssd");
169+
170+
bool embedded_processing;
171+
embedded_processing = utils::get_from_any_maps("embedded_processing", config, {}, embedded_processing);
172+
EXPECT_TRUE(embedded_processing);
173+
}
174+
175+
TEST_P(SSDModelParameterizedTest, TestDetectionCustomConfig) {
176+
GTEST_SKIP() << "Detection config tests fail on CI";
177+
auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
178+
std::vector<std::string> mock_labels;
179+
size_t num_classes = 80;
180+
for (size_t i = 0; i < num_classes; i++) {
181+
mock_labels.push_back(std::to_string(i));
182+
}
183+
ov::AnyMap configuration = {{"layout", "data:HWC"}, {"resize_type", "fit_to_window"}, {"labels", mock_labels}};
184+
bool preload = true;
185+
auto model = DetectionModel::create_model(DATA_DIR + "/" + model_path, configuration, preload, "CPU");
186+
187+
auto config = model.algorithm->adapter->getModelConfig();
188+
std::string layout;
189+
layout = utils::get_from_any_maps("layout", config, {}, layout);
190+
EXPECT_EQ(layout, configuration.at("layout").as<std::string>());
191+
192+
std::string resize_type;
193+
resize_type = utils::get_from_any_maps("resize_type", config, {}, resize_type);
194+
EXPECT_EQ(resize_type, configuration.at("resize_type").as<std::string>());
195+
196+
std::vector<std::string> labels;
197+
labels = utils::get_from_any_maps("labels", config, {}, labels);
198+
for (size_t i = 0; i < num_classes; i++) {
199+
EXPECT_EQ(labels[i], mock_labels[i]);
200+
}
201+
}
202+
203+
//TEST_P(DetectionModelParameterizedTestSaveLoad, TestDetctionCorrectnessAfterSaveLoad) {
204+
// cv::Mat image = cv::imread(DATA_DIR + "/" + IMAGE_PATH);
205+
// if (!image.data) {
206+
// throw std::runtime_error{"Failed to read the image"};
207+
// }
208+
//
209+
// auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
210+
// bool preload = true;
211+
// auto model = DetectionModel::create_model(DATA_DIR + "/" + model_path, {}, preload, "CPU");
212+
//
213+
// auto ov_model = model->getModel();
214+
// ov::serialize(ov_model, TMP_MODEL_FILE);
215+
//
216+
// auto result = model->infer(image)->objects;
217+
//
218+
// image = cv::imread(DATA_DIR + "/" + IMAGE_PATH);
219+
// if (!image.data) {
220+
// throw std::runtime_error{"Failed to read the image"};
221+
// }
222+
// auto model_restored = DetectionModel::create_model(TMP_MODEL_FILE, {}, "", preload, "CPU");
223+
// auto result_data = model_restored->infer(image);
224+
// auto result_restored = result_data->objects;
225+
//
226+
// ASSERT_EQ(result.size(), result_restored.size());
227+
//
228+
// for (size_t i = 0; i < result.size(); i++) {
229+
// ASSERT_EQ(result[i].x, result_restored[i].x);
230+
// ASSERT_EQ(result[i].y, result_restored[i].y);
231+
// ASSERT_EQ(result[i].width, result_restored[i].width);
232+
// ASSERT_EQ(result[i].height, result_restored[i].height);
233+
// }
234+
//}
235+
//
236+
//TEST_P(DetectionModelParameterizedTestSaveLoad, TestDetctionCorrectnessAfterSaveLoadWithAdapter) {
237+
// cv::Mat image = cv::imread(DATA_DIR + "/" + IMAGE_PATH);
238+
// if (!image.data) {
239+
// throw std::runtime_error{"Failed to read the image"};
240+
// }
241+
//
242+
// auto model_path = string_format(MODEL_PATH_TEMPLATE, GetParam().name.c_str(), GetParam().name.c_str());
243+
// bool preload = true;
244+
// auto model = DetectionModel::create_model(DATA_DIR + "/" + model_path, {}, "", preload, "CPU");
245+
// auto ov_model = model->getModel();
246+
// ov::serialize(ov_model, TMP_MODEL_FILE);
247+
// auto result = model->infer(image)->objects;
248+
//
249+
// image = cv::imread(DATA_DIR + "/" + IMAGE_PATH);
250+
// if (!image.data) {
251+
// throw std::runtime_error{"Failed to read the image"};
252+
// }
253+
//
254+
// std::shared_ptr<InferenceAdapter> adapter = std::make_shared<MockAdapter>(TMP_MODEL_FILE);
255+
// auto model_restored = DetectionModel::create_model(adapter);
256+
// auto result_data = model_restored->infer(image);
257+
// auto result_restored = result_data->objects;
258+
//
259+
// ASSERT_EQ(result.size(), result_restored.size());
260+
//
261+
// for (size_t i = 0; i < result.size(); i++) {
262+
// ASSERT_EQ(result[i].x, result_restored[i].x);
263+
// ASSERT_EQ(result[i].y, result_restored[i].y);
264+
// ASSERT_EQ(result[i].width, result_restored[i].width);
265+
// ASSERT_EQ(result[i].height, result_restored[i].height);
266+
// }
267+
//}
268+
269+
INSTANTIATE_TEST_SUITE_P(ClassificationTestInstance,
270+
ClassificationModelParameterizedTest,
271+
::testing::Values(ModelData("mlc_mobilenetv3_large_voc")));
272+
//INSTANTIATE_TEST_SUITE_P(ClassificationTestInstance,
273+
// ClassificationModelParameterizedTestSaveLoad,
274+
// ::testing::Values(ModelData("mlc_mobilenetv3_large_voc")));
275+
INSTANTIATE_TEST_SUITE_P(SSDTestInstance,
276+
SSDModelParameterizedTest,
277+
::testing::Values(ModelData("detection_model_with_xai_head")));
278+
//INSTANTIATE_TEST_SUITE_P(SSDTestInstance,
279+
// DetectionModelParameterizedTestSaveLoad,
280+
// ::testing::Values(ModelData("detection_model_with_xai_head")));
281+
282+
class InputParser {
283+
public:
284+
InputParser(int& argc, char** argv) {
285+
for (int i = 1; i < argc; ++i)
286+
this->tokens.push_back(std::string(argv[i]));
287+
}
288+
289+
const std::string& getCmdOption(const std::string& option) const {
290+
std::vector<std::string>::const_iterator itr;
291+
itr = std::find(this->tokens.begin(), this->tokens.end(), option);
292+
if (itr != this->tokens.end() && ++itr != this->tokens.end()) {
293+
return *itr;
294+
}
295+
static const std::string empty_string("");
296+
return empty_string;
297+
}
298+
299+
bool cmdOptionExists(const std::string& option) const {
300+
return std::find(this->tokens.begin(), this->tokens.end(), option) != this->tokens.end();
301+
}
302+
303+
private:
304+
std::vector<std::string> tokens;
305+
};
306+
307+
void print_help(const char* program_name) {
308+
std::cout << "Usage: " << program_name << "-d <path_to_data>" << std::endl;
309+
}
310+
311+
int main(int argc, char** argv) {
312+
InputParser input(argc, argv);
313+
314+
if (input.cmdOptionExists("-h")) {
315+
print_help(argv[0]);
316+
return 1;
317+
}
318+
319+
const std::string& data_dir = input.getCmdOption("-d");
320+
if (!data_dir.empty()) {
321+
DATA_DIR = data_dir;
322+
} else {
323+
print_help(argv[0]);
324+
return 1;
325+
}
326+
327+
testing::InitGoogleTest(&argc, argv);
328+
329+
return RUN_ALL_TESTS();
330+
}

0 commit comments

Comments
 (0)