diff --git a/demos/classification_benchmark_demo/cpp/main.cpp b/demos/classification_benchmark_demo/cpp/main.cpp index 42dbb6d3e74..3db6c86b03d 100644 --- a/demos/classification_benchmark_demo/cpp/main.cpp +++ b/demos/classification_benchmark_demo/cpp/main.cpp @@ -51,6 +51,7 @@ static const char target_device_message[] = "Optional. Specify the target device static const char num_threads_message[] = "Optional. Specify count of threads."; static const char num_streams_message[] = "Optional. Specify count of streams."; static const char num_inf_req_message[] = "Optional. Number of infer requests."; +static const char num_inf_req_per_batch_message[] = "Optional. Number of infer requests per batch."; static const char image_grid_resolution_message[] = "Optional. Set image grid resolution in format WxH. " "Default value is 1280x720."; static const char ntop_message[] = "Optional. Number of top results. Default value is 5. Must be >= 1."; @@ -75,6 +76,7 @@ DEFINE_string(d, "CPU", target_device_message); DEFINE_uint32(nthreads, 0, num_threads_message); DEFINE_string(nstreams, "", num_streams_message); DEFINE_uint32(nireq, 0, num_inf_req_message); +DEFINE_uint32(nireq_per_batch, 1, num_inf_req_per_batch_message); DEFINE_uint32(nt, 5, ntop_message); DEFINE_string(res, "1280x720", image_grid_resolution_message); DEFINE_bool(auto_resize, false, input_resizable_message); @@ -84,6 +86,7 @@ DEFINE_string(u, "", utilization_monitors_message); DEFINE_bool(reverse_input_channels, false, reverse_input_channels_message); DEFINE_string(mean_values, "", mean_values_message); DEFINE_string(scale_values, "", scale_values_message); +DEFINE_string(config, "", "Path to the configuration file (optional)"); static void showUsage() { std::cout << std::endl; @@ -109,6 +112,7 @@ static void showUsage() { std::cout << " -reverse_input_channels " << reverse_input_channels_message << std::endl; std::cout << " -mean_values " << mean_values_message << std::endl; std::cout << " -scale_values " << scale_values_message << std::endl; + std::cout << " -config " << "Path to config file" << std::endl; } bool ParseAndCheckCommandLine(int argc, char* argv[]) { @@ -135,6 +139,44 @@ bool ParseAndCheckCommandLine(int argc, char* argv[]) { return true; } + +std::map parseConfigFile() { + std::map config; + + std::ifstream file(FLAGS_config); + if(!file.is_open()) { + std::cerr << "Can't open file " << FLAGS_config << " for read" << std::endl; + exit(-1); + } + + std::string option; + while (std::getline(file, option)) { + if (option.empty() || option[0] == '#') { + continue; + } + size_t spacePos = option.find_first_of(" \t\n\r"); + if(spacePos == std::string::npos) { + std::cerr << "Invalid config parameter format. Space separator required here: " << option; + exit(-1); + } + + std::string key, value; + if (spacePos != std::string::npos) { + key = option.substr(0, spacePos); + size_t valueStart = option.find_first_not_of(" \t\n\r", spacePos); + if(valueStart == std::string::npos) { + std::cerr << "An invalid config parameter value detected, it mustn't be empty: " << option; + exit(-1); + } + size_t valueEnd = option.find_last_not_of(" \t\n\r"); + value = option.substr(valueStart, valueEnd - valueStart + 1); + config[key] = value; + } + } + + return config; +} + cv::Mat centerSquareCrop(const cv::Mat& image) { if (image.cols >= image.rows) { return image(cv::Rect((image.cols - image.rows) / 2, 0, image.rows, image.rows)); @@ -233,6 +275,10 @@ int main(int argc, char* argv[]) { slog::info << ov::get_openvino_version() << slog::endl; ov::Core core; + if (!FLAGS_config.empty()) { + const auto configs = parseConfigFile(); + core.set_property(FLAGS_d, {configs.begin(), configs.end()}); + } std::unique_ptr model(new ClassificationModel(FLAGS_m, FLAGS_nt, FLAGS_auto_resize, labels, FLAGS_layout)); model->setInputsPreprocessing(FLAGS_reverse_input_channels, FLAGS_mean_values, FLAGS_scale_values); @@ -265,6 +311,22 @@ int main(int argc, char* argv[]) { std::size_t nextImageIndex = 0; std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); + // batch setup + std::vector> inputDataVector; + std::transform(inputImages.begin(), inputImages.end(), std::back_inserter(inputDataVector), [](const auto &src) { + return std::make_shared(src); + }); + auto inputImagesDataBeginIt = inputDataVector.begin(); + auto inputImagesDataEndIt = inputImagesDataBeginIt; + std::advance(inputImagesDataEndIt, FLAGS_nireq_per_batch); + + auto inputImagesBeginIt = inputImages.begin(); + auto inputImagesEndIt = inputImagesBeginIt; + std::advance(inputImagesEndIt, FLAGS_nireq_per_batch); + + auto classIndicesBeginIt = classIndices.begin(); + auto classIndicesEndIt = classIndicesBeginIt; + std::advance(classIndicesEndIt, FLAGS_nireq_per_batch); while (keepRunning && elapsedSeconds < std::chrono::seconds(FLAGS_time)) { if (elapsedSeconds >= testDuration - fpsCalculationDuration && framesNumOnCalculationStart == 0) { framesNumOnCalculationStart = framesNum; @@ -287,14 +349,36 @@ int main(int argc, char* argv[]) { if (pipeline.isReadyToProcess()) { auto imageStartTime = std::chrono::steady_clock::now(); - pipeline.submitData(ImageInputData(inputImages[nextImageIndex]), - std::make_shared(inputImages[nextImageIndex], + pipeline.submitData(inputImagesDataBeginIt, inputImagesDataEndIt, + std::make_shared(inputImagesBeginIt, inputImagesEndIt, imageStartTime, - classIndices[nextImageIndex])); - nextImageIndex++; - if (nextImageIndex == imageNames.size()) { - nextImageIndex = 0; + classIndicesBeginIt, classIndicesEndIt)); + //nextImageIndex++; + //if (nextImageIndex == imageNames.size()) { + //nextImageIndex = 0; + //} + + ++inputImagesDataBeginIt; + ++inputImagesDataEndIt; + ++inputImagesBeginIt; + ++inputImagesEndIt; + ++classIndicesBeginIt; + ++classIndicesEndIt; + + if (inputImagesEndIt == inputImages.end()) { + inputImagesBeginIt = inputImages.begin(); + inputImagesEndIt = inputImagesBeginIt; + std::advance(inputImagesEndIt, FLAGS_nireq_per_batch); + + inputImagesDataBeginIt = inputDataVector.begin(); + inputImagesDataEndIt = inputImagesDataBeginIt; + std::advance(inputImagesDataEndIt, FLAGS_nireq_per_batch); + + classIndicesBeginIt = classIndices.begin(); + classIndicesEndIt = classIndicesBeginIt; + std::advance(classIndicesEndIt, FLAGS_nireq_per_batch); } + } //--- Waiting for free input slot or output data available. Function will return immediately if any of them @@ -308,58 +392,61 @@ int main(int argc, char* argv[]) { if (!classificationResult.metaData) { throw std::invalid_argument("Renderer: metadata is null"); } - const ClassificationImageMetaData& classificationImageMetaData = - classificationResult.metaData->asRef(); - - auto outputImg = classificationImageMetaData.img; - - if (outputImg.empty()) { - throw std::invalid_argument("Renderer: image provided in metadata is empty"); - } - PredictionResult predictionResult = PredictionResult::Incorrect; - std::string label = classificationResult.topLabels.front().label; - if (!FLAGS_gt.empty()) { - for (size_t i = 0; i < FLAGS_nt; i++) { - unsigned predictedClass = classificationResult.topLabels[i].id; - if (predictedClass == classificationImageMetaData.groundTruthId) { - predictionResult = PredictionResult::Correct; - correctPredictionsCount++; - label = classificationResult.topLabels[i].label; - break; - } + const ClassificationImageBatchMetaData& classificationImageBatchMetaData = + classificationResult.metaData->asRef(); + + //auto outputImg = classificationImageMetaData.img; + const std::vector> &outputImagesMD = classificationImageBatchMetaData.metadatas; + for (const std::shared_ptr &classificationImageMetaData : outputImagesMD) { + auto outputImg = classificationImageMetaData->img; + if (outputImg.empty()) { + throw std::invalid_argument("Renderer: image provided in metadata is empty"); } - } else { - predictionResult = PredictionResult::Unknown; - } - framesNum++; - gridMat.updateMat(outputImg, label, predictionResult); - accuracy = static_cast(correctPredictionsCount) / framesNum; - gridMat.textUpdate(metrics, - classificationResult.metaData->asRef().timeStamp, - accuracy, - FLAGS_nt, - isTestMode, - !FLAGS_gt.empty(), - presenter); - renderMetrics.update(renderingStart); - elapsedSeconds = std::chrono::steady_clock::now() - startTime; - if (!FLAGS_no_show) { - cv::imshow("classification_demo", gridMat.outImg); - //--- Processing keyboard events - int key = cv::waitKey(1); - if (27 == key || 'q' == key || 'Q' == key) { // Esc - keepRunning = false; - } else if (32 == key || 'r' == key || - 'R' == key) { // press space or r to restart testing if needed - isTestMode = true; - framesNum = 0; - framesNumOnCalculationStart = 0; - correctPredictionsCount = 0; - accuracy = 0; - elapsedSeconds = std::chrono::steady_clock::duration(0); - startTime = std::chrono::steady_clock::now(); + PredictionResult predictionResult = PredictionResult::Incorrect; + std::string label = classificationResult.topLabels.front().label; + if (!FLAGS_gt.empty()) { + for (size_t i = 0; i < FLAGS_nt; i++) { + unsigned predictedClass = classificationResult.topLabels[i].id; + if (predictedClass == classificationImageMetaData->groundTruthId) { + predictionResult = PredictionResult::Correct; + correctPredictionsCount++; + label = classificationResult.topLabels[i].label; + break; + } + } } else { - presenter.handleKey(key); + predictionResult = PredictionResult::Unknown; + } + framesNum += 1; + gridMat.updateMat(outputImg, label, predictionResult); + accuracy = static_cast(correctPredictionsCount) / framesNum; + gridMat.textUpdate(metrics, + classificationImageMetaData->timeStamp, + accuracy, + FLAGS_nt, + isTestMode, + !FLAGS_gt.empty(), + presenter); + renderMetrics.update(renderingStart); + elapsedSeconds = std::chrono::steady_clock::now() - startTime; + if (!FLAGS_no_show) { + cv::imshow("classification_demo", gridMat.outImg); + //--- Processing keyboard events + int key = cv::waitKey(1); + if (27 == key || 'q' == key || 'Q' == key) { // Esc + keepRunning = false; + } else if (32 == key || 'r' == key || + 'R' == key) { // press space or r to restart testing if needed + isTestMode = true; + framesNum = 0; + framesNumOnCalculationStart = 0; + correctPredictionsCount = 0; + accuracy = 0; + elapsedSeconds = std::chrono::steady_clock::duration(0); + startTime = std::chrono::steady_clock::now(); + } else { + presenter.handleKey(key); + } } } } diff --git a/demos/common/cpp/models/include/models/detection_model_ssd.h b/demos/common/cpp/models/include/models/detection_model_ssd.h index 5fe34dfd66d..da0990fd9b4 100644 --- a/demos/common/cpp/models/include/models/detection_model_ssd.h +++ b/demos/common/cpp/models/include/models/detection_model_ssd.h @@ -50,6 +50,9 @@ class ModelSSD : public DetectionModel { const std::string& layout = ""); std::shared_ptr preprocess(const InputData& inputData, ov::InferRequest& request) override; + std::shared_ptr preprocess(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + ov::InferRequest& request) override; std::unique_ptr postprocess(InferenceResult& infResult) override; protected: diff --git a/demos/common/cpp/models/include/models/image_model.h b/demos/common/cpp/models/include/models/image_model.h index aa8ab609b9e..df8ceb40384 100644 --- a/demos/common/cpp/models/include/models/image_model.h +++ b/demos/common/cpp/models/include/models/image_model.h @@ -38,7 +38,9 @@ class ImageModel : public ModelBase { ImageModel(const std::string& modelFileName, bool useAutoResize, const std::string& layout = ""); std::shared_ptr preprocess(const InputData& inputData, ov::InferRequest& request) override; - + std::shared_ptr preprocess(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + ov::InferRequest& request) override; protected: bool useAutoResize; diff --git a/demos/common/cpp/models/include/models/model_base.h b/demos/common/cpp/models/include/models/model_base.h index 00e5a3bc104..d86dba8ae21 100644 --- a/demos/common/cpp/models/include/models/model_base.h +++ b/demos/common/cpp/models/include/models/model_base.h @@ -40,6 +40,9 @@ class ModelBase { virtual ~ModelBase() {} virtual std::shared_ptr preprocess(const InputData& inputData, ov::InferRequest& request) = 0; + virtual std::shared_ptr preprocess(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + ov::InferRequest& request) {return {};}; virtual ov::CompiledModel compileModel(const ModelConfig& config, ov::Core& core); virtual void onLoadCompleted(const std::vector& requests) {} virtual std::unique_ptr postprocess(InferenceResult& infResult) = 0; diff --git a/demos/common/cpp/models/src/classification_model.cpp b/demos/common/cpp/models/src/classification_model.cpp index be80d897747..89e20c0b6b7 100644 --- a/demos/common/cpp/models/src/classification_model.cpp +++ b/demos/common/cpp/models/src/classification_model.cpp @@ -55,7 +55,7 @@ std::unique_ptr ClassificationModel::postprocess(InferenceResult& in for (size_t i = 0; i < scoresTensor.get_size(); ++i) { int ind = indicesPtr[i]; if (ind < 0 || ind >= static_cast(labels.size())) { - throw std::runtime_error("Invalid index for the class label is found during postprocessing"); + throw std::runtime_error(std::string("Invalid index: ") + std::to_string(ind) + " for the class label is found during postprocessing, label size: " + std::to_string(labels.size())); } result->topLabels.emplace_back(ind, labels[ind], scoresPtr[i]); } diff --git a/demos/common/cpp/models/src/detection_model_ssd.cpp b/demos/common/cpp/models/src/detection_model_ssd.cpp index 992b41ffe6a..394c422ac93 100644 --- a/demos/common/cpp/models/src/detection_model_ssd.cpp +++ b/demos/common/cpp/models/src/detection_model_ssd.cpp @@ -53,6 +53,21 @@ std::shared_ptr ModelSSD::preprocess(const InputData& inputDa return DetectionModel::preprocess(inputData, request); } +std::shared_ptr ModelSSD::preprocess(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + ov::InferRequest& request) { + if (inputsNames.size() > 1) { + const auto& imageInfoTensor = request.get_tensor(inputsNames[1]); + const auto info = imageInfoTensor.data(); + info[0] = static_cast(netInputHeight); + info[1] = static_cast(netInputWidth); + info[2] = 1; + request.set_tensor(inputsNames[1], imageInfoTensor); + } + + return DetectionModel::preprocess(inputDataBegin, inputDataEnd, request); +} + std::unique_ptr ModelSSD::postprocess(InferenceResult& infResult) { return outputsNames.size() > 1 ? postprocessMultipleOutputs(infResult) : postprocessSingleOutput(infResult); } diff --git a/demos/common/cpp/models/src/image_model.cpp b/demos/common/cpp/models/src/image_model.cpp index 817c94d9c79..94541d737dd 100644 --- a/demos/common/cpp/models/src/image_model.cpp +++ b/demos/common/cpp/models/src/image_model.cpp @@ -32,6 +32,67 @@ ImageModel::ImageModel(const std::string& modelFileName, bool useAutoResize, con : ModelBase(modelFileName, layout), useAutoResize(useAutoResize) {} +std::shared_ptr ImageModel::preprocess(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + ov::InferRequest& request) { + + const ov::Tensor& frameTensor = request.get_tensor(inputsNames[0]); // first input should be image + const ov::Shape& tensorShape = frameTensor.get_shape(); + const ov::Layout layout("NHWC"); + const size_t batch = tensorShape[ov::layout::batch_idx(layout)]; + const size_t width = tensorShape[ov::layout::width_idx(layout)]; + const size_t height = tensorShape[ov::layout::height_idx(layout)]; + const size_t channels = tensorShape[ov::layout::channels_idx(layout)]; + char* memoryBlob = nullptr; + size_t image_index = 0; + bool isMatFloat = false; + int origImg_cols = 0; + int origImg_rows = 0; + size_t image_count = std::distance(inputDataBegin, inputDataEnd); + if (image_count != batch) { + throw std::runtime_error("Image count in preprocess must repeat batch count"); + } + for (auto inputDataIt = inputDataBegin; inputDataIt != inputDataEnd; ++inputDataIt ) { + const auto& origImg = (*inputDataIt)->asRef().inputImage; + origImg_cols = origImg.cols; + origImg_rows = origImg.rows; + auto img = inputTransform(origImg); + + auto matType = img.type() & CV_MAT_DEPTH_MASK; + if (matType != CV_8U && matType != CV_32F) { + throw std::runtime_error("Unsupported mat type for wrapping"); + } + isMatFloat = matType == CV_32F; + + if (!useAutoResize) { + // /* Resize and copy data from the image to the input tensor */ + + if (static_cast(img.channels()) != channels) { + throw std::runtime_error(std::string("The number of channels for model input: ") + + std::to_string(channels) + " and image: " + + std::to_string(img.channels()) + " - must match"); + } + if (channels != 1 && channels != 3) { + throw std::runtime_error("Unsupported number of channels"); + } + img = resizeImageExt(img, width, height, resizeMode, interpolationMode); + } + size_t sizeInBytes = img.total() * img.elemSize(); + if (!memoryBlob) { + memoryBlob = new char[sizeInBytes * batch]; // intended memory leak + } + + // fill continuous batch + memcpy(memoryBlob + sizeInBytes * image_index, img.ptr(), sizeInBytes); + image_index++; + } + + auto precision = isMatFloat ? ov::element::f32 : ov::element::u8; + auto batched_tensor = ov::Tensor(precision, ov::Shape{ batch, height, width, channels }, memoryBlob); + request.set_tensor(inputsNames[0], batched_tensor); + return std::make_shared(origImg_cols, origImg_rows); +} + std::shared_ptr ImageModel::preprocess(const InputData& inputData, ov::InferRequest& request) { const auto& origImg = inputData.asRef().inputImage; auto img = inputTransform(origImg); diff --git a/demos/common/cpp/pipelines/include/pipelines/async_pipeline.h b/demos/common/cpp/pipelines/include/pipelines/async_pipeline.h index 308c65cc8b8..52c74eee542 100644 --- a/demos/common/cpp/pipelines/include/pipelines/async_pipeline.h +++ b/demos/common/cpp/pipelines/include/pipelines/async_pipeline.h @@ -75,6 +75,9 @@ class AsyncPipeline { /// Otherwise returns unique sequential frame ID for this particular request. Same frame ID will be written in the /// result structure. virtual int64_t submitData(const InputData& inputData, const std::shared_ptr& metaData); + virtual int64_t submitData(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + const std::shared_ptr& metaData); /// Gets available data from the queue /// @param shouldKeepOrder if true, function will treat results as ready only if next sequential result (frame) is diff --git a/demos/common/cpp/pipelines/include/pipelines/metadata.h b/demos/common/cpp/pipelines/include/pipelines/metadata.h index 4bbe1456a47..2466110a1a0 100644 --- a/demos/common/cpp/pipelines/include/pipelines/metadata.h +++ b/demos/common/cpp/pipelines/include/pipelines/metadata.h @@ -40,6 +40,31 @@ struct ImageMetaData : public MetaData { ImageMetaData(cv::Mat img, std::chrono::steady_clock::time_point timeStamp) : img(img), timeStamp(timeStamp) {} }; +struct ImageBatchMetaData : public MetaData { + std::chrono::steady_clock::time_point timeStamp; + std::vector> metadatas; + + ImageBatchMetaData() {} + + ImageBatchMetaData(std::vector::iterator imagesBeginIt, + const std::vector::iterator imagesEndIt, + std::chrono::steady_clock::time_point timeStamp) : timeStamp(timeStamp) { + size_t images_count = std::distance(imagesBeginIt, imagesEndIt); + metadatas.reserve(images_count); + for (; imagesBeginIt != imagesEndIt;) { + metadatas.push_back(std::make_shared(*imagesBeginIt++, timeStamp)); + } + } + + void add(cv::Mat img, std::chrono::steady_clock::time_point timeStamp) { + metadatas.push_back(std::make_shared(img, timeStamp)); + this->timeStamp = timeStamp; + } + void clear() { + metadatas.clear(); + } +}; + struct ClassificationImageMetaData : public ImageMetaData { unsigned int groundTruthId; @@ -49,3 +74,26 @@ struct ClassificationImageMetaData : public ImageMetaData { : ImageMetaData(img, timeStamp), groundTruthId(groundTruthId) {} }; + + +struct ClassificationImageBatchMetaData : public MetaData { + std::vector> metadatas; + + ClassificationImageBatchMetaData(std::vector::iterator imagesBeginIt, + const std::vector::iterator imagesEndIt, + std::chrono::steady_clock::time_point timeStamp, + std::vector::iterator groundTruthIdsBeginIt, + const std::vector::iterator groundTruthIdsEndIt) + : MetaData(){ + size_t images_count = std::distance(imagesBeginIt, imagesEndIt); + size_t gt_count = std::distance(groundTruthIdsBeginIt, groundTruthIdsEndIt); + if (images_count != gt_count) { + throw std::runtime_error("images.size() != groundTruthIds.size()"); + } + + metadatas.reserve(images_count); + for (; imagesBeginIt != imagesEndIt;) { + metadatas.push_back(std::make_shared(*imagesBeginIt++, timeStamp, *groundTruthIdsBeginIt++)); + } + } +}; diff --git a/demos/common/cpp/pipelines/src/async_pipeline.cpp b/demos/common/cpp/pipelines/src/async_pipeline.cpp index 42ff6bbd7fe..34efb937ec4 100644 --- a/demos/common/cpp/pipelines/src/async_pipeline.cpp +++ b/demos/common/cpp/pipelines/src/async_pipeline.cpp @@ -76,6 +76,60 @@ void AsyncPipeline::waitForData(bool shouldKeepOrder) { } } +int64_t AsyncPipeline::submitData(std::vector>::iterator inputDataBegin, + std::vector>::iterator inputDataEnd, + const std::shared_ptr& metaData) { + auto frameID = inputFrameId; + + auto request = requestsPool->getIdleRequest(); + if (!request) { + return -1; + } + + auto startTime = std::chrono::steady_clock::now(); + auto internalModelData = model->preprocess(inputDataBegin, inputDataEnd, request); + preprocessMetrics.update(startTime); + + request.set_callback( + [this, request, frameID, internalModelData, metaData, startTime](std::exception_ptr ex) mutable { + { + const std::lock_guard lock(mtx); + inferenceMetrics.update(startTime); + try { + if (ex) { + std::rethrow_exception(ex); + } + InferenceResult result; + + result.frameId = frameID; + result.metaData = std::move(metaData); + result.internalModelData = std::move(internalModelData); + + for (const auto& outName : model->getOutputsNames()) { + auto tensor = request.get_tensor(outName); + result.outputsData.emplace(outName, tensor); + } + + completedInferenceResults.emplace(frameID, result); + requestsPool->setRequestIdle(request); + } catch (...) { + if (!callbackException) { + callbackException = std::current_exception(); + } + } + } + condVar.notify_one(); + }); + + inputFrameId++; + if (inputFrameId < 0) + inputFrameId = 0; + + request.start_async(); + + return frameID; +} + int64_t AsyncPipeline::submitData(const InputData& inputData, const std::shared_ptr& metaData) { auto frameID = inputFrameId; diff --git a/demos/object_detection_demo/cpp/main.cpp b/demos/object_detection_demo/cpp/main.cpp index b69ceffbd2a..97cad98b21e 100644 --- a/demos/object_detection_demo/cpp/main.cpp +++ b/demos/object_detection_demo/cpp/main.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,7 @@ static const char mean_values_message[] = "Optional. Normalize input by subtracting the mean values per channel. Example: \"255.0 255.0 255.0\""; static const char scale_values_message[] = "Optional. Divide input by scale values per channel. Division is applied " "after mean values subtraction. Example: \"255.0 255.0 255.0\""; +static const char num_inf_req_per_batch_message[] = "Optional. "; DEFINE_bool(h, false, help_message); DEFINE_string(at, "", at_message); @@ -115,6 +117,7 @@ DEFINE_bool(r, false, raw_output_message); DEFINE_double(t, 0.5, thresh_output_message); DEFINE_double(iou_t, 0.5, iou_thresh_output_message); DEFINE_bool(auto_resize, false, input_resizable_message); +DEFINE_uint32(nireq_per_batch, 1, num_inf_req_per_batch_message); DEFINE_uint32(nireq, 0, nireq_message); DEFINE_uint32(nthreads, 0, num_threads_message); DEFINE_string(nstreams, "", num_streams_message); @@ -127,6 +130,7 @@ DEFINE_string(masks, "", masks_message); DEFINE_bool(reverse_input_channels, false, reverse_input_channels_message); DEFINE_string(mean_values, "", mean_values_message); DEFINE_string(scale_values, "", scale_values_message); +DEFINE_string(config, "", "Path to the configuration file (optional)"); /** * \brief This function shows a help message @@ -262,6 +266,100 @@ bool ParseAndCheckCommandLine(int argc, char* argv[]) { return true; } + +std::map parseConfigFile() { + std::map config; + + std::ifstream file(FLAGS_config); + if(!file.is_open()) { + std::cerr << "Can't open file " << FLAGS_config << " for read" << std::endl; + exit(-1); + } + + std::string option; + while (std::getline(file, option)) { + if (option.empty() || option[0] == '#') { + continue; + } + size_t spacePos = option.find_first_of(" \t\n\r"); + if(spacePos == std::string::npos) { + std::cerr << "Invalid config parameter format. Space separator required here: " << option; + exit(-1); + } + + std::string key, value; + if (spacePos != std::string::npos) { + key = option.substr(0, spacePos); + size_t valueStart = option.find_first_not_of(" \t\n\r", spacePos); + if(valueStart == std::string::npos) { + std::cerr << "An invalid config parameter value detected, it mustn't be empty: " << option; + exit(-1); + } + size_t valueEnd = option.find_last_not_of(" \t\n\r"); + value = option.substr(valueStart, valueEnd - valueStart + 1); + config[key] = value; + } + } + + return config; +} + + +std::vector renderDetectionBatchData(DetectionResult& result, const ColorPalette& palette, OutputTransform& outputTransform) { + if (!result.metaData) { + throw std::invalid_argument("Renderer: metadata is null"); + } + + const auto &batchedMetadata = result.metaData->asRef(); + std::vector outputs; + outputs.reserve(batchedMetadata.metadatas.size()); + for (auto &metadata : batchedMetadata.metadatas) { + auto outputImg = metadata->img; + + if (outputImg.empty()) { + throw std::invalid_argument("Renderer: image provided in metadata is empty"); + } + outputTransform.resize(outputImg); + // Visualizing result data over source image + if (FLAGS_r) { + slog::debug << " -------------------- Frame # " << result.frameId << "--------------------" << slog::endl; + slog::debug << " Class ID | Confidence | XMIN | YMIN | XMAX | YMAX " << slog::endl; + } + + for (auto& obj : result.objects) { + if (FLAGS_r) { + slog::debug << " " << std::left << std::setw(9) << obj.label << " | " << std::setw(10) << obj.confidence + << " | " << std::setw(4) << int(obj.x) << " | " << std::setw(4) << int(obj.y) << " | " + << std::setw(4) << int(obj.x + obj.width) << " | " << std::setw(4) << int(obj.y + obj.height) + << slog::endl; + } + outputTransform.scaleRect(obj); + std::ostringstream conf; + conf << ":" << std::fixed << std::setprecision(1) << obj.confidence * 100 << '%'; + const auto& color = palette[obj.labelID]; + putHighlightedText(outputImg, + obj.label + conf.str(), + cv::Point2f(obj.x, obj.y - 5), + cv::FONT_HERSHEY_COMPLEX_SMALL, + 1, + color, + 2); + cv::rectangle(outputImg, obj, color, 2); + } + + try { + for (auto& lmark : result.asRef().landmarks) { + outputTransform.scaleCoord(lmark); + cv::circle(outputImg, lmark, 2, cv::Scalar(0, 255, 255), -1); + } + } catch (const std::bad_cast&) {} + outputs.push_back(outputImg); + } + return outputs; +} + + + // Input image is stored inside metadata, as we put it there during submission stage cv::Mat renderDetectionData(DetectionResult& result, const ColorPalette& palette, OutputTransform& outputTransform) { if (!result.metaData) { @@ -399,6 +497,10 @@ int main(int argc, char* argv[]) { slog::info << ov::get_openvino_version() << slog::endl; ov::Core core; + if (!FLAGS_config.empty()) { + const auto configs = parseConfigFile(); + core.set_property(FLAGS_d, {configs.begin(), configs.end()}); + } AsyncPipeline pipeline(std::move(model), ConfigFactory::getUserConfig(FLAGS_d, FLAGS_nireq, FLAGS_nstreams, FLAGS_nthreads), @@ -418,20 +520,33 @@ int main(int argc, char* argv[]) { OutputTransform outputTransform = OutputTransform(); size_t found = FLAGS_output_resolution.find("x"); + // batch setup + std::vector> inputDataVector; + inputDataVector.reserve(FLAGS_nireq_per_batch); + auto image_batch_metadata = std::make_shared(); while (keepRunning) { if (pipeline.isReadyToProcess()) { auto startTime = std::chrono::steady_clock::now(); - //--- Capturing frame - curr_frame = cap->read(); - - if (curr_frame.empty()) { - // Input stream is over + //--- Capturing nireq_per_batch frames + inputDataVector.clear(); + image_batch_metadata->clear(); + for (int i = 0; i < FLAGS_nireq_per_batch; i++) { + curr_frame = cap->read(); + if (curr_frame.empty()) { + // Input stream is over + break; + } + inputDataVector.push_back(std::make_shared(curr_frame)); + image_batch_metadata->add(curr_frame, startTime); + } + if (inputDataVector.size() != FLAGS_nireq_per_batch) { break; } - - frameNum = pipeline.submitData(ImageInputData(curr_frame), - std::make_shared(curr_frame, startTime)); + auto inputImagesDataBeginIt = inputDataVector.begin(); + auto inputImagesDataEndIt = inputImagesDataBeginIt; + std::advance(inputImagesDataEndIt, FLAGS_nireq_per_batch); + frameNum = pipeline.submitData(inputImagesDataBeginIt, inputImagesDataEndIt, image_batch_metadata); } if (frameNum == 0) { @@ -455,28 +570,30 @@ int main(int argc, char* argv[]) { // and use your own processing instead of calling renderDetectionData(). while (keepRunning && (result = pipeline.getResult())) { auto renderingStart = std::chrono::steady_clock::now(); - cv::Mat outFrame = renderDetectionData(result->asRef(), palette, outputTransform); + std::vector outFrames = renderDetectionBatchData(result->asRef(), palette, outputTransform); //--- Showing results and device information - presenter.drawGraphs(outFrame); - renderMetrics.update(renderingStart); - metrics.update(result->metaData->asRef().timeStamp, - outFrame, - {10, 22}, - cv::FONT_HERSHEY_COMPLEX, - 0.65); - - videoWriter.write(outFrame); - framesProcessed++; - - if (!FLAGS_no_show) { - cv::imshow("Detection Results", outFrame); - //--- Processing keyboard events - int key = cv::waitKey(1); - if (27 == key || 'q' == key || 'Q' == key) { // Esc - keepRunning = false; - } else { - presenter.handleKey(key); + for (cv::Mat outFrame : outFrames) { + presenter.drawGraphs(outFrame); + renderMetrics.update(renderingStart); + metrics.update(result->metaData->asRef().timeStamp, + outFrame, + {10, 22}, + cv::FONT_HERSHEY_COMPLEX, + 0.65); + + videoWriter.write(outFrame); + framesProcessed++; + + if (!FLAGS_no_show) { + cv::imshow("Detection Results", outFrame); + //--- Processing keyboard events + int key = cv::waitKey(1); + if (27 == key || 'q' == key || 'Q' == key) { // Esc + keepRunning = false; + } else { + presenter.handleKey(key); + } } } } @@ -489,20 +606,22 @@ int main(int argc, char* argv[]) { result = pipeline.getResult(); if (result != nullptr) { auto renderingStart = std::chrono::steady_clock::now(); - cv::Mat outFrame = renderDetectionData(result->asRef(), palette, outputTransform); + std::vector outFrames = renderDetectionBatchData(result->asRef(), palette, outputTransform); //--- Showing results and device information - presenter.drawGraphs(outFrame); - renderMetrics.update(renderingStart); - metrics.update(result->metaData->asRef().timeStamp, - outFrame, - {10, 22}, - cv::FONT_HERSHEY_COMPLEX, - 0.65); - videoWriter.write(outFrame); - if (!FLAGS_no_show) { - cv::imshow("Detection Results", outFrame); - //--- Updating output window - cv::waitKey(1); + for (cv::Mat outFrame : outFrames) { + presenter.drawGraphs(outFrame); + renderMetrics.update(renderingStart); + metrics.update(result->metaData->asRef().timeStamp, + outFrame, + {10, 22}, + cv::FONT_HERSHEY_COMPLEX, + 0.65); + videoWriter.write(outFrame); + if (!FLAGS_no_show) { + cv::imshow("Detection Results", outFrame); + //--- Updating output window + cv::waitKey(1); + } } } }