Integrate Intel OpenVINO options into compiler plugin

Use the options API to the OpenVINO compiler plugin for compiling the
Openvino compiled_model. The compiler now applies user-specified
settings for device selection, performance hints, and custom properties.

Signed-off-by: Anoob Anto Kodankandath <[email protected]>

Author: akodanka

litert/tools/BUILD

@@ -139,6 +139,7 @@ litert_bin(
         "//litert/tools/flags:common_flags",
         "//litert/tools/flags:flag_types",
         "//litert/tools/flags/vendors:google_tensor_flags_with_dynamic_runtime",
+        "//litert/tools/flags/vendors:intel_openvino_flags_with_dynamic_runtime",
         "//litert/tools/flags/vendors:mediatek_flags_with_dynamic_runtime",
         "//litert/tools/flags/vendors:qualcomm_flags_with_dynamic_runtime",
         "@com_google_absl//absl/flags:flag",
@@ -258,6 +259,7 @@ cc_binary(
         "//litert/tools/flags:common_flags",
         "//litert/tools/flags:flag_types",
         "//litert/tools/flags/vendors:google_tensor_flags_with_dynamic_runtime",
+        "//litert/tools/flags/vendors:intel_openvino_flags_with_dynamic_runtime",
         "//litert/tools/flags/vendors:mediatek_flags_with_dynamic_runtime",
         "//litert/tools/flags/vendors:qualcomm_flags_with_dynamic_runtime",
         "@com_google_absl//absl/flags:flag",
@@ -446,6 +448,7 @@ cc_binary(
         "//litert/cc:litert_options",
         "//litert/cc:litert_tensor_buffer",
         "//litert/tools/flags/vendors:google_tensor_flags",
+        "//litert/tools/flags/vendors:intel_openvino_flags",
         "//litert/tools/flags/vendors:mediatek_flags",
         "//litert/tools/flags/vendors:qualcomm_flags",
         "//tflite/profiling:time",

litert/tools/apply_plugin_main.cc

@@ -14,6 +14,7 @@
 
 #define INCLUDE_QUALCOMM_COMPILE_FLAGS
 #define INCLUDE_MEDIATEK_COMPILE_FLAGS
+#define INCLUDE_INTEL_OPENVINO_COMPILE_FLAGS
 #define INCLUDE_GOOGLE_TENSOR_COMPILE_FLAGS
 
 #include <memory>
@@ -30,6 +31,7 @@
 #include "litert/tools/flags/common_flags.h"
 #include "litert/tools/flags/flag_types.h"
 #include "litert/tools/flags/vendors/google_tensor_flags.h"  // IWYU pragma: keep
+#include "litert/tools/flags/vendors/intel_openvino_flags.h"  // IWYU pragma: keep
 #include "litert/tools/flags/vendors/mediatek_flags.h"  // IWYU pragma: keep
 #include "litert/tools/flags/vendors/qualcomm_flags.h"  // IWYU pragma: keep
 #include "litert/tools/outstream.h"
@@ -136,6 +138,21 @@ int main(int argc, char* argv[]) {
     }
   }
 
+  {
+    auto intel_openvino_opts =
+        litert::intel_openvino::IntelOpenVinoOptionsFromFlags();
+    if (!intel_openvino_opts) {
+      run->dump_out.Get().get() << "Failed to create Intel OpenVINO options\n";
+      return 1;
+    }
+
+    if (!opts->AddOpaqueOptions(std::move(*intel_openvino_opts))) {
+      run->dump_out.Get().get()
+          << "Failed to add Intel OpenVINO options to list\n";
+      return 1;
+    }
+  }
+
   {
     auto mediatek_opts =
         litert::mediatek::MediatekOptionsFromFlags();

litert/tools/run_model.cc

@@ -14,6 +14,7 @@
 
 #define INCLUDE_QUALCOMM_RUNTIME_FLAGS
 #define INCLUDE_MEDIATEK_RUNTIME_FLAGS
+#define INCLUDE_INTEL_OPENVINO_RUNTIME_FLAGS
 #define INCLUDE_GOOGLE_TENSOR_RUNTIME_FLAGS
 
 #include <algorithm>
@@ -43,6 +44,7 @@
 #include "litert/cc/litert_options.h"
 #include "litert/cc/litert_tensor_buffer.h"
 #include "litert/tools/flags/vendors/google_tensor_flags.h"  // IWYU pragma: keep
+#include "litert/tools/flags/vendors/intel_openvino_flags.h"  // IWYU pragma: keep
 #include "litert/tools/flags/vendors/mediatek_flags.h"  // IWYU pragma: keep
 #include "litert/tools/flags/vendors/qualcomm_flags.h"  // IWYU pragma: keep
 #include "litert/tools/tensor_utils.h"
@@ -74,6 +76,7 @@ namespace litert {
 namespace {
 
 using ::litert::google_tensor::GoogleTensorOptionsFromFlags;
+using ::litert::intel_openvino::IntelOpenVinoOptionsFromFlags;
 using ::litert::mediatek::MediatekOptionsFromFlags;
 using ::litert::qualcomm::QualcommOptionsFromFlags;
 
@@ -122,6 +125,9 @@ Expected<Options> GetOptions() {
   if (auto google_tensor_opts = GoogleTensorOptionsFromFlags()) {
     options.AddOpaqueOptions(std::move(*google_tensor_opts));
   }
+  if (auto intel_openvino_opts = IntelOpenVinoOptionsFromFlags()) {
+    options.AddOpaqueOptions(std::move(*intel_openvino_opts));
+  }
   if (auto mediatek_opts = MediatekOptionsFromFlags()) {
     options.AddOpaqueOptions(std::move(*mediatek_opts));
   }

litert/vendors/intel_openvino/compiler/decoder.cc

@@ -194,6 +194,7 @@ constexpr const char* GetOvOpType(const LiteRtOpCode op_code) {
   for (const auto& entry : kLitertOvMap) {
     if (entry.first == op_code) return entry.second;
   }
+  LITERT_LOG(LITERT_WARNING, "op_code(%d) not supported", op_code);
   return "";
 }
 

litert/vendors/intel_openvino/compiler/openvino_compiler_plugin.cc

@@ -35,8 +35,11 @@
 #include "litert/cc/litert_expected.h"
 #include "litert/cc/litert_macros.h"
 #include "litert/cc/litert_model.h"
+#include "litert/cc/options/litert_intel_openvino_options.h"
 #include "litert/vendors/c/litert_compiler_plugin.h"
+#include "litert/vendors/cc/options_helper.h"
 #include "litert/vendors/intel_openvino/compiler/graph_iterator.h"
+
 namespace {
 
 constexpr char kPluginManufacturer[] = "IntelOpenVINO";
@@ -216,17 +219,35 @@ LiteRtStatus LiteRtCompiledResultNumByteCodeModules(
 // Plugin Definition
 /// \brief Define Compiler plugin APIs
 struct LiteRtCompilerPluginT {
-  LiteRtEnvironmentOptions env;
-  LiteRtOptions options;
+  using IntelOpenVinoOptions = ::litert::intel_openvino::IntelOpenVinoOptions;
+
+  LiteRtCompilerPluginT(LiteRtEnvironmentOptions env, LiteRtOptions options) {
+    std::tie(env_, opts_, opq_, intel_openvino_opts_) =
+        litert::ParseOptions<IntelOpenVinoOptions>(env, options);
+  }
+
+  const ::litert::Expected<IntelOpenVinoOptions>& GetIntelOpenVinoOptions() const {
+    return intel_openvino_opts_;
+  }
+
+  const ::litert::Expected<litert::OpaqueOptions>& GetOpaqueOptions() const { return opq_; }
+
+ private:
+  litert::Expected<litert::EnvironmentOptions> env_ = litert::Error(
+      kLiteRtStatusErrorInvalidArgument, "Null environment options");
+  litert::Expected<litert::Options> opts_ =
+      litert::Error(kLiteRtStatusErrorInvalidArgument, "Null options");
+  litert::Expected<litert::OpaqueOptions> opq_ =
+      litert::Error(kLiteRtStatusErrorInvalidArgument, "Null opaque options");
+  litert::Expected<IntelOpenVinoOptions> intel_openvino_opts_ =
+      litert::Error(kLiteRtStatusErrorInvalidArgument, "Null Intel OpenVINO options");
 };
 
 LiteRtStatus LiteRtCreateCompilerPlugin(LiteRtCompilerPlugin *compiler_plugin,
                                         LiteRtEnvironmentOptions env,
                                         LiteRtOptions options) {
   LiteRtSetMinLoggerSeverity(LiteRtGetDefaultLogger(), LITERT_INFO);
-  auto *plugin = new LiteRtCompilerPluginT;
-  plugin->env = env;
-  plugin->options = options;
+  auto *plugin = new LiteRtCompilerPluginT(env, options);
   *compiler_plugin = plugin;
   return kLiteRtStatusOk;
 }
@@ -273,13 +294,77 @@ LiteRtStatus LiteRtCompilerPluginCompile(
     auto model = litert::Model::CreateFromNonOwnedHandle(partitions);
     const auto num_partitions = model.NumSubgraphs();
 
+    // Configure device and OpenVINO settings from Intel OpenVINO options
+    std::string device = "NPU";  // Default device
+    ov::AnyMap configs_map;
+
+    if (compiler_plugin->GetIntelOpenVinoOptions().HasValue()) {
+      const auto& intel_opts = compiler_plugin->GetIntelOpenVinoOptions().Value();
+
+      // Configure device type
+      auto device_type = intel_opts.GetDeviceType();
+      switch (device_type) {
+        case kLiteRtIntelOpenVinoDeviceTypeCPU:
+          device = "CPU";
+          break;
+        case kLiteRtIntelOpenVinoDeviceTypeGPU:
+          device = "GPU";
+          break;
+        case kLiteRtIntelOpenVinoDeviceTypeNPU:
+          device = "NPU";
+          break;
+        case kLiteRtIntelOpenVinoDeviceTypeAUTO:
+          device = "AUTO";
+          break;
+      }
+
+      LITERT_LOG(LITERT_INFO, "Using Intel OpenVINO device: %s", device.c_str());
+
+      auto performance_mode = intel_opts.GetPerformanceMode();
+
+      // Add custom configuration options
+      int num_custom_options = intel_opts.GetNumConfigsMapOptions();
+      for (int i = 0; i < num_custom_options; ++i) {
+        auto [key, value] = intel_opts.GetConfigsMapOption(i);
+        if (!key.empty()) {  // Valid config option
+          configs_map[key] = value;
+          LITERT_LOG(LITERT_INFO, "Custom config: %s = %s", key.c_str(), value.c_str());
+        }
+      }
+
+      // Configure performance mode (can be overridden by custom options)
+      switch (performance_mode) {
+        case kLiteRtIntelOpenVinoPerformanceModeLatency:
+          if (configs_map.find(ov::hint::performance_mode.name()) == configs_map.end()) {
+            configs_map[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::LATENCY;
+            LITERT_LOG(LITERT_INFO, "Performance mode: LATENCY");
+          }
+          break;
+        case kLiteRtIntelOpenVinoPerformanceModeThroughput:
+          if (configs_map.find(ov::hint::performance_mode.name()) == configs_map.end()) {
+            configs_map[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::THROUGHPUT;
+            LITERT_LOG(LITERT_INFO, "Performance mode: THROUGHPUT");
+          }
+          break;
+        case kLiteRtIntelOpenVinoPerformanceModeCumulativeThroughput:
+          if (configs_map.find(ov::hint::performance_mode.name()) == configs_map.end()) {
+            configs_map[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT;
+            LITERT_LOG(LITERT_INFO, "Performance mode: CUMULATIVE_THROUGHPUT");
+          }
+          break;
+      }
+    } else {
+      // Default configuration if no options provided
+      configs_map[ov::hint::performance_mode.name()] = ov::hint::PerformanceMode::LATENCY;
+      LITERT_LOG(LITERT_INFO, "Using default configuration (LATENCY mode)");
+    }
+
     auto result = std::make_unique<LiteRtCompiledResultT>();
     result->byte_code.resize(num_partitions);
     result->graph_names.resize(num_partitions);
     auto tflite_fe =
         std::make_shared<ov::frontend::tensorflow_lite::FrontEnd>();
-    // TODO: Update this hard coded path to an env option passed from LiteRT
-    // framework
+
     ov::Core core;
     for (int partition_idx = 0; partition_idx < num_partitions;
          ++partition_idx) {
@@ -295,10 +380,9 @@ LiteRtStatus LiteRtCompilerPluginCompile(
         LITERT_LOG(LITERT_INFO, "Model loaded");
         auto model = tflite_fe->convert(input_model);
 
-        // TODO: pass the device string from env options
-        std::string device = "NPU";
+        // Use device and configs_map from Intel OpenVINO options
         std::ostringstream oss;
-        auto compiled_model = core.compile_model(model, device);
+        auto compiled_model = core.compile_model(model, device, configs_map);
         compiled_model.export_model(oss);
         LITERT_LOG(LITERT_INFO, "Model export done");
         result->byte_code[partition_idx] = oss.str();