E#-88506 Extend G-API classification benchmark to work with different inference backends (#3853)

* Add gapi execution provider selection

* Make compiled with OCV 4.9

* Add GAPI inference backend selection, Apply some review comments

* Add checks on backend arguments

* Apply review comments, Add MACRO guards for different backends employment

* Fix -backend help message

* Fix compilation bug with provider template

* Fix possible 'unusable params' warning

* Update demos/classification_benchmark_demo/cpp_gapi/README.md

Co-authored-by: Zlobin Vladimir <[email protected]>

* Fix comments, Add improved openCV version check

* Add mean/scale into README.md

* Update demos/classification_benchmark_demo/cpp_gapi/README.md

applied

Co-authored-by: Zlobin Vladimir <[email protected]>

* Add missing ONNX mean/scale processing

* Decrease IE and ONNX in opencv support versions requirements

---------

Co-authored-by: Zlobin Vladimir <[email protected]>

Author: sivanov-work

demos/classification_benchmark_demo/cpp_gapi/README.md

@@ -145,6 +145,9 @@ Options:
     -no_show                  Optional. Disable showing of processed images.
     -time "<integer>"         Optional. Time in seconds to execute program. Default is -1 (infinite time).
     -u                        Optional. List of monitors to show initially.
+    -backend <string>         Optional. Specify an inference backend. The list of available backends depends on openCV version. Default value is IE. See README.md for details
+    -mean_values              Optional. Normalize input by subtracting the mean values per channel. Example: "255.0 255.0 255.0"
+    -scale_values             Optional. Divide input by scale values per channel. Division is applied after mean values subtraction. Example: "255.0 255.0 255.0"
 ```
 
 The number of `InferRequest`s is specified by -nireq flag. Each `InferRequest` acts as a "buffer": it waits in queue before being filled with images and sent for inference, then after the inference completes, it waits in queue until its results are processed. Increasing the number of `InferRequest`s usually increases performance, because in that case multiple `InferRequest`s can be processed simultaneously if the device supports parallelization. However, big number of `InferRequest`s increases latency because each image still needs to wait in queue.
@@ -161,6 +164,10 @@ For example, use the following command-line command to run the application:
                       -u CDM
 ```
 
+To let the demo find onnx libs copy `onnxruntime_providers_openvino.dll` from `lib` and `onnxruntime.dll`, `onnxruntime_providers_shared.dll` from `bin` dirs  of onnxruntume install to the folder with the demo executable.
+
+The inference backend is specified by -backend flag. Depends on openCV version there might be different backends available: `IE` starting from openCV 4.2.0, `ONNX` from 4.5.1 and `OV` from 4.8. The possible device value for `"ONNX/DML/<device>` can be found using `set OPENCV_LOG_LEVEL=INFO` and running the demo with `-backend ONNX/DML/`. Search for `Available DirectML adapters:`. For `-backend ONNX/OV/<device>` the device is the same as the one selected for `--use_openvino` while compiling `onnxruntime`.
+
 ## Demo Output
 
 The demo uses OpenCV to display the resulting image grid with classification results presented as a text above images. The demo reports:

demos/classification_benchmark_demo/cpp_gapi/classification_benchmark_demo_gapi.hpp

@@ -8,6 +8,9 @@
 
 #include <gflags/gflags.h>
 
+#include <utils_gapi/backend_description.hpp>
+#include <utils/args_helper.hpp>
+
 static const char help_message[] = "Print a usage message.";
 static const char image_message[] = "Required. Path to a folder with images or path to an image file.";
 static const char model_message[] = "Required. Path to an .xml file with a trained model.";
@@ -26,7 +29,13 @@ static const char no_show_message[] = "Optional. Disable showing of processed im
 static const char execution_time_message[] = "Optional. Time in seconds to execute program. "
                                              "Default is -1 (infinite time).";
 static const char utilization_monitors_message[] = "Optional. List of monitors to show initially.";
-
+static const std::string backend_message_str("Optional. Specify an inference backend. The list of available backends: " +
+                                             merge(getSupportedInferenceBackends(), ",") + ". Default value is IE. See README.md for details");
+static const char *backend_message = backend_message_str.c_str();
+static const char mean_values_message[] =
+    "Optional. Normalize input by subtracting the mean values per channel. Example: \"255.0 255.0 255.0\"";
+static const char scale_values_message[] = "Optional. Divide input by scale values per channel. Division is applied "
+                                           "after mean values subtraction. Example: \"255.0 255.0 255.0\"";
 
 DEFINE_bool(h, false, help_message);
 DEFINE_string(i, "", image_message);
@@ -42,6 +51,9 @@ DEFINE_string(res, "1280x720", image_grid_resolution_message);
 DEFINE_bool(no_show, false, no_show_message);
 DEFINE_uint32(time, std::numeric_limits<gflags::uint32>::max(), execution_time_message);
 DEFINE_string(u, "", utilization_monitors_message);
+DEFINE_string(backend, "IE", backend_message);
+DEFINE_string(mean_values, "", mean_values_message);
+DEFINE_string(scale_values, "", scale_values_message);
 
 /**
  * \brief This function shows a help message
@@ -66,4 +78,7 @@ static void showUsage() {
     std::cout << "    -no_show                  " << no_show_message << std::endl;
     std::cout << "    -time \"<integer>\"         " << execution_time_message << std::endl;
     std::cout << "    -u                        " << utilization_monitors_message << std::endl;
+    std::cout << "    -backend                  " << backend_message << std::endl;
+    std::cout << "    -mean_values              " << mean_values_message << std::endl;
+    std::cout << "    -scale_values             " << scale_values_message << std::endl;
 }

demos/classification_benchmark_demo/cpp_gapi/main.cpp

@@ -18,8 +18,6 @@
 #include <opencv2/gapi/gcomputation.hpp>
 #include <opencv2/gapi/gmat.hpp>
 #include <opencv2/gapi/gproto.hpp>
-#include <opencv2/gapi/infer.hpp>
-#include <opencv2/gapi/infer/ie.hpp>
 #include <opencv2/gapi/util/optional.hpp>
 
 
@@ -36,11 +34,15 @@
 #include <utils/slog.hpp>
 #include <utils_gapi/kernel_package.hpp>
 #include <utils_gapi/stream_source.hpp>
+#include <utils_gapi/backend_builder.hpp>
 
 #include "classification_benchmark_demo_gapi.hpp"
 #include "custom_kernels.hpp"
 #include <models/classification_model.h>
 
+namespace nets {
+G_API_NET(Classification, <cv::GMat(cv::GMat)>, "classification");
+}
 
 namespace util {
 bool ParseAndCheckCommandLine(int argc, char* argv[]) {
@@ -61,11 +63,16 @@ bool ParseAndCheckCommandLine(int argc, char* argv[]) {
     return true;
 }
 
-}  // namespace util
-
-namespace nets {
-G_API_NET(Classification, <cv::GMat(cv::GMat)>, "classification");
+inference_backends_t ParseInferenceBackends(const std::string &str, char sep = ',') {
+    inference_backends_t backends;
+    std::stringstream params_list(str);
+    std::string line;
+    while (std::getline(params_list, line, sep)) {
+        backends.push(BackendDescription::parseFromArgs(line));
+    }
+    return backends;
 }
+}  // namespace util
 
 int main(int argc, char* argv[]) {
     try {
@@ -133,19 +140,16 @@ int main(int argc, char* argv[]) {
         });
 
         /** Configure network **/
+        auto nets = cv::gapi::networks();
         auto config = ConfigFactory::getUserConfig(FLAGS_d, FLAGS_nireq, FLAGS_nstreams, FLAGS_nthreads);
-        // clang-format off
-        const auto net =
-            cv::gapi::ie::Params<nets::Classification>{
-                FLAGS_m,  // path to topology IR
-                fileNameNoExt(FLAGS_m) + ".bin",  // path to weights
-                FLAGS_d  // device specifier
-            }.cfgNumRequests(config.maxAsyncRequests)
-             .pluginConfig(config.getLegacyConfig());
-        // clang-format on
-
+        inference_backends_t backends = util::ParseInferenceBackends(FLAGS_backend);
+        nets += create_execution_network<nets::Classification>(FLAGS_m,
+                                                               BackendsConfig {config,
+                                                                              FLAGS_mean_values,
+                                                                              FLAGS_scale_values},
+                                                               backends);
         auto pipeline = comp.compileStreaming(cv::compile_args(custom::kernels(),
-                                              cv::gapi::networks(net),
+                                              nets,
                                               cv::gapi::streaming::queue_capacity{1}));
 
         /** Output container for result **/

demos/common/cpp/utils/include/utils/args_helper.hpp

@@ -32,6 +32,9 @@ void readInputFilesArguments(std::vector<std::string>& files, const std::string&
 void parseInputFilesArguments(std::vector<std::string>& files);
 
 std::vector<std::string> split(const std::string& s, char delim);
+void split(const std::string& s, char delim, std::vector<float> &out);
+std::string merge(std::initializer_list<std::string> list, const char *delim);
+std::string merge(const std::vector<std::string> &list, const char *delim);
 
 std::vector<std::string> parseDevices(const std::string& device_string);
 

demos/common/cpp/utils/include/utils/config_factory.h

@@ -31,7 +31,7 @@ struct ModelConfig {
     ov::AnyMap compiledModelConfig;
 
     std::set<std::string> getDevices();
-    std::map<std::string, std::string> getLegacyConfig();
+    std::map<std::string, std::string> getLegacyConfig() const;
 
 protected:
     std::set<std::string> devices;

demos/common/cpp/utils/src/args_helper.cpp

@@ -17,6 +17,7 @@
 #include <map>
 
 #include <algorithm>
+#include <iterator>
 #include <cctype>
 #include <sstream>
 
@@ -77,6 +78,37 @@ std::vector<std::string> split(const std::string& s, char delim) {
     return result;
 }
 
+void split(const std::string& s, char delim, std::vector<float> &out) {
+    std::stringstream ss(s);
+    std::string item;
+
+    while (getline(ss, item, delim)) {
+        try {
+            out.push_back(std::stof(item));
+        } catch (...) {
+            throw std::runtime_error("cannot split the string: \"" + s + "\" onto floats");
+        }
+    }
+}
+
+template <class It>
+static std::string merge_impl(It begin, It end, const char* delim) {
+    std::stringstream ss;
+    std::copy(begin, end, std::ostream_iterator<std::string>(ss, delim));
+    std::string result = ss.str();
+    if (!result.empty()) {
+        result.resize(result.size() - strlen(delim));
+    }
+    return result;
+}
+std::string merge(std::initializer_list<std::string> list, const char* delim) {
+    return merge_impl(list.begin(), list.end(), delim);
+}
+
+std::string merge(const std::vector<std::string> &list, const char *delim) {
+    return merge_impl(list.begin(), list.end(), delim);
+}
+
 std::vector<std::string> parseDevices(const std::string& device_string) {
     const std::string::size_type colon_position = device_string.find(":");
     if (colon_position != std::string::npos) {

demos/common/cpp/utils/src/config_factory.cpp

@@ -93,7 +93,7 @@ ModelConfig ConfigFactory::getCommonConfig(const std::string& flags_d, uint32_t
     return config;
 }
 
-std::map<std::string, std::string> ModelConfig::getLegacyConfig() {
+std::map<std::string, std::string> ModelConfig::getLegacyConfig() const {
     std::map<std::string, std::string> config;
     for (const auto& item : compiledModelConfig) {
         config[item.first] = item.second.as<std::string>();

demos/common/cpp_gapi/utils_gapi/include/utils_gapi/backend_builder.hpp

@@ -0,0 +1,49 @@
+// Copyright (C) 2021-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <functional>
+#include <list>
+#include <map>
+#include <stdexcept>
+#include <string>
+
+#include "ie_backend.hpp"
+#include "onnx_backend.hpp"
+#include "ov_backend.hpp"
+
+template<class ExecNetwork>
+cv::gapi::GNetPackage create_execution_network(const std::string &model_path,
+                                               const BackendsConfig &config,
+                                               const inference_backends_t &backends = inference_backends_t{}) {
+    if (backends.empty()) {
+        throw std::runtime_error("No G-API backend specified.\nPlease select a backend from the list: " +
+                                 merge(getSupportedInferenceBackends(), ", "));
+    }
+    static const std::map<std::string,
+                          std::function<cv::gapi::GNetPackage(const std::string &,
+                                                              const BackendsConfig &,
+                                                              const inference_backends_t &)>
+                         > maps {
+#ifdef GAPI_IE_BACKEND
+        {"IE",      &BackendApplicator<ExecNetwork, cv::gapi::ie::Params>::apply}
+#endif
+#ifdef GAPI_ONNX_BACKEND
+        , { "ONNX", &BackendApplicator<ExecNetwork, cv::gapi::onnx::Params>::apply}
+#endif
+#ifdef GAPI_OV_BACKEND
+        , { "OV",   &BackendApplicator<ExecNetwork, cv::gapi::ov::Params>::apply}
+#endif
+    };
+
+    const BackendDescription &backend = backends.front();
+    const auto it = maps.find(backend.name);
+    if (it == maps.end()) {
+        throw std::runtime_error("Cannot apply unknown G-API backend: " + backend.name +
+                                 "\nPlease, check on available backend list: " +
+                                 merge(getSupportedInferenceBackends(), ","));
+    }
+    return it->second(model_path, config, backends);
+}

demos/common/cpp_gapi/utils_gapi/include/utils_gapi/backend_description.hpp

@@ -0,0 +1,43 @@
+// Copyright (C) 2021-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <queue>
+#include <string>
+#include <vector>
+
+#include <opencv2/gapi/infer.hpp>
+#include <utils/config_factory.h>
+
+#include "utils_gapi/gapi_features.hpp"
+
+struct BackendDescription {
+    template<class It>
+    BackendDescription(const std::string &name, It begin, It end) :
+        name(name), properties(begin, end) {}
+
+    static BackendDescription parseFromArgs(const std::string &arg, char sep = '/');
+
+    std::string name;
+    std::vector<std::string> properties;
+};
+
+struct BackendsConfig: ModelConfig {
+    BackendsConfig(const ModelConfig &src,
+                   const std::string &mean_values = "",
+                   const std::string &scale_values = "");
+    std::string mean_values;
+    std::string scale_values;
+};
+
+using inference_backends_t = std::queue<BackendDescription>;
+
+std::initializer_list<std::string> getSupportedInferenceBackends();
+
+template<class ExecNetwork,
+         template <class> class Params>
+struct BackendApplicator {
+    static cv::gapi::GNetPackage apply(const std::string&, const BackendsConfig &, const inference_backends_t &);
+};

demos/common/cpp_gapi/utils_gapi/include/utils_gapi/gapi_features.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <opencv2/core/version.hpp>
+#include <opencv2/gapi/core.hpp>
+
+#if CV_VERSION_MAJOR > 4 || (CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR >= 2)
+    #define GAPI_IE_BACKEND
+#endif
+
+#if CV_VERSION_MAJOR > 4 || (CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR > 5)
+    #define GAPI_ONNX_BACKEND
+#endif
+
+#if CV_VERSION_MAJOR > 4 || (CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR >= 8)
+    #define GAPI_OV_BACKEND
+#endif
+
+#if CV_VERSION_MAJOR > 4 || (CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR > 8)
+    #define GAPI_ONNX_BACKEND_EP_EXTENSION
+#endif