Skip to content

Commit d1daedd

Browse files
akodankaakodanka
committed
Add Intel OpenVINO options API for LiteRT
Implement C and C++ APIs for configuring Intel OpenVINO compiler settings. Includes options for device selection, performance tuning, and custom configuration properties. Documentation: litert/vendors/intel_openvino/README_options.md Sample usage: litert/vendors/intel_openvino/intel_openvino_options_example.cc Signed-off-by: Anoob Anto Kodankandath <[email protected]>
1 parent 0f264fe commit d1daedd

File tree

11 files changed

+897
-0
lines changed

11 files changed

+897
-0
lines changed

‎litert/c/options/BUILD‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,21 @@ cc_library(
130130
],
131131
)
132132

133+
# INTEL OPENVINO ##############################################################################
134+
135+
cc_library(
136+
name = "litert_intel_openvino_options",
137+
srcs = ["litert_intel_openvino_options.cc"],
138+
hdrs = ["litert_intel_openvino_options.h"],
139+
deps = [
140+
"//litert/c:litert_common",
141+
"//litert/c:litert_opaque_options",
142+
"//litert/cc:litert_macros",
143+
"//litert/core/cache:hash_util",
144+
"@com_google_absl//absl/strings:string_view",
145+
],
146+
)
147+
133148
cc_library(
134149
name = "litert_gpu_options",
135150
srcs = ["litert_gpu_options.cc"],

‎litert/c/options/litert_intel_openvino_options.cc‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
// Copyright (C) 2025 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
#include "litert/c/options/litert_intel_openvino_options.h"
17+
18+
#include <cstdint>
19+
#include <memory>
20+
#include <string>
21+
#include <utility>
22+
#include <vector>
23+
24+
#include "absl/strings/string_view.h" // from @com_google_absl
25+
#include "litert/c/litert_common.h"
26+
#include "litert/c/litert_opaque_options.h"
27+
#include "litert/cc/litert_macros.h"
28+
#include "litert/core/cache/hash_util.h"
29+
30+
struct LiteRtIntelOpenVinoOptionsT {
31+
LiteRtIntelOpenVinoDeviceType device_type = kLiteRtIntelOpenVinoDeviceTypeNPU;
32+
LiteRtIntelOpenVinoPerformanceMode performance_mode =
33+
kLiteRtIntelOpenVinoPerformanceModeLatency;
34+
// Store custom configuration options as key-value pairs
35+
std::vector<std::pair<std::string, std::string>> configs_map_options;
36+
};
37+
38+
LiteRtStatus LiteRtIntelOpenVinoOptionsCreate(LiteRtOpaqueOptions* options) {
39+
if (options == nullptr) {
40+
return kLiteRtStatusErrorInvalidArgument;
41+
}
42+
43+
auto options_data = std::make_unique<LiteRtIntelOpenVinoOptionsT>();
44+
45+
LITERT_RETURN_IF_ERROR(LiteRtCreateOpaqueOptions(
46+
LiteRtIntelOpenVinoOptionsGetIdentifier(), options_data.get(),
47+
[](void* payload) {
48+
delete reinterpret_cast<LiteRtIntelOpenVinoOptions>(payload);
49+
},
50+
options));
51+
52+
auto intel_openvino_hash = [](const void* payload) -> uint64_t {
53+
const LiteRtIntelOpenVinoOptionsT* opts =
54+
reinterpret_cast<const LiteRtIntelOpenVinoOptionsT*>(payload);
55+
uint64_t ans = 0;
56+
litert::HashCombine(ans, opts->device_type, opts->performance_mode);
57+
// Hash the configs_map_options
58+
for (const auto& pair : opts->configs_map_options) {
59+
litert::HashCombine(ans, pair.first, pair.second);
60+
}
61+
return ans;
62+
};
63+
LITERT_RETURN_IF_ERROR(LiteRtSetOpaqueOptionsHash(*options, intel_openvino_hash));
64+
65+
options_data.release();
66+
return kLiteRtStatusOk;
67+
}
68+
69+
const char* LiteRtIntelOpenVinoOptionsGetIdentifier() {
70+
return "intel_openvino";
71+
}
72+
73+
LiteRtStatus LiteRtIntelOpenVinoOptionsGet(LiteRtOpaqueOptions options,
74+
LiteRtIntelOpenVinoOptions* options_data) {
75+
if (options_data == nullptr || options == nullptr) {
76+
return kLiteRtStatusErrorInvalidArgument;
77+
}
78+
const char* identifier;
79+
LITERT_RETURN_IF_ERROR(
80+
LiteRtGetOpaqueOptionsIdentifier(options, &identifier));
81+
if (absl::NullSafeStringView(identifier) !=
82+
LiteRtIntelOpenVinoOptionsGetIdentifier()) {
83+
return kLiteRtStatusErrorInvalidArgument;
84+
}
85+
void* payload;
86+
LITERT_RETURN_IF_ERROR(LiteRtGetOpaqueOptionsData(options, &payload));
87+
*options_data = reinterpret_cast<LiteRtIntelOpenVinoOptionsT*>(payload);
88+
return kLiteRtStatusOk;
89+
}
90+
91+
// COMPILATION OPTIONS /////////////////////////////////////////////////////////
92+
93+
// device_type ----------------------------------------------------------------
94+
LiteRtStatus LiteRtIntelOpenVinoOptionsSetDeviceType(
95+
LiteRtIntelOpenVinoOptions options,
96+
LiteRtIntelOpenVinoDeviceType device_type) {
97+
if (options == nullptr) {
98+
return kLiteRtStatusErrorInvalidArgument;
99+
}
100+
options->device_type = device_type;
101+
return kLiteRtStatusOk;
102+
}
103+
104+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetDeviceType(
105+
LiteRtIntelOpenVinoOptions options,
106+
LiteRtIntelOpenVinoDeviceType* device_type) {
107+
if (options == nullptr || device_type == nullptr) {
108+
return kLiteRtStatusErrorInvalidArgument;
109+
}
110+
*device_type = options->device_type;
111+
return kLiteRtStatusOk;
112+
}
113+
114+
// performance_mode -----------------------------------------------------------
115+
LiteRtStatus LiteRtIntelOpenVinoOptionsSetPerformanceMode(
116+
LiteRtIntelOpenVinoOptions options,
117+
LiteRtIntelOpenVinoPerformanceMode performance_mode) {
118+
if (options == nullptr) {
119+
return kLiteRtStatusErrorInvalidArgument;
120+
}
121+
options->performance_mode = performance_mode;
122+
return kLiteRtStatusOk;
123+
}
124+
125+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetPerformanceMode(
126+
LiteRtIntelOpenVinoOptions options,
127+
LiteRtIntelOpenVinoPerformanceMode* performance_mode) {
128+
if (options == nullptr || performance_mode == nullptr) {
129+
return kLiteRtStatusErrorInvalidArgument;
130+
}
131+
*performance_mode = options->performance_mode;
132+
return kLiteRtStatusOk;
133+
}
134+
135+
// configs_map_options --------------------------------------------------------
136+
LiteRtStatus LiteRtIntelOpenVinoOptionsSetConfigsMapOption(
137+
LiteRtIntelOpenVinoOptions options,
138+
const char* key,
139+
const char* value) {
140+
if (options == nullptr || key == nullptr || value == nullptr) {
141+
return kLiteRtStatusErrorInvalidArgument;
142+
}
143+
// Check if the key already exists and update it, otherwise add new
144+
for (auto& pair : options->configs_map_options) {
145+
if (pair.first == key) {
146+
pair.second = value;
147+
return kLiteRtStatusOk;
148+
}
149+
}
150+
options->configs_map_options.emplace_back(key, value);
151+
return kLiteRtStatusOk;
152+
}
153+
154+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetNumConfigsMapOptions(
155+
LiteRtIntelOpenVinoOptions options,
156+
int* num_options) {
157+
if (options == nullptr || num_options == nullptr) {
158+
return kLiteRtStatusErrorInvalidArgument;
159+
}
160+
*num_options = static_cast<int>(options->configs_map_options.size());
161+
return kLiteRtStatusOk;
162+
}
163+
164+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetConfigsMapOption(
165+
LiteRtIntelOpenVinoOptions options,
166+
int index,
167+
const char** key,
168+
const char** value) {
169+
if (options == nullptr || key == nullptr || value == nullptr) {
170+
return kLiteRtStatusErrorInvalidArgument;
171+
}
172+
if (index < 0 || index >= static_cast<int>(options->configs_map_options.size())) {
173+
return kLiteRtStatusErrorIndexOOB;
174+
}
175+
const auto& pair = options->configs_map_options[index];
176+
*key = pair.first.c_str();
177+
*value = pair.second.c_str();
178+
return kLiteRtStatusOk;
179+
}

‎litert/c/options/litert_intel_openvino_options.h‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Copyright (C) 2025 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
#ifndef THIRD_PARTY_ODML_LITERT_LITERT_C_OPTIONS_LITERT_INTEL_OPENVINO_OPTIONS_H_
17+
#define THIRD_PARTY_ODML_LITERT_LITERT_C_OPTIONS_LITERT_INTEL_OPENVINO_OPTIONS_H_
18+
#include "litert/c/litert_common.h"
19+
#include "litert/c/litert_opaque_options.h"
20+
21+
#ifdef __cplusplus
22+
extern "C" {
23+
#endif // __cplusplus
24+
25+
// Create an Intel OpenVINO options object that is type erased. The actual option
26+
// data can be accessed from the payload.
27+
LiteRtStatus LiteRtIntelOpenVinoOptionsCreate(LiteRtOpaqueOptions* options);
28+
LITERT_DEFINE_HANDLE(LiteRtIntelOpenVinoOptions);
29+
30+
// The string identifier that discriminates Intel OpenVINO options within
31+
// type erased options.
32+
const char* LiteRtIntelOpenVinoOptionsGetIdentifier();
33+
34+
// Attempt to retrieve Intel OpenVINO options from the opaque options. Fails
35+
// if the opaque options are of another type.
36+
LiteRtStatus LiteRtIntelOpenVinoOptionsGet(LiteRtOpaqueOptions options,
37+
LiteRtIntelOpenVinoOptions* options_data);
38+
39+
// COMPILATION OPTIONS /////////////////////////////////////////////////////////
40+
41+
// device_type ----------------------------------------------------------------
42+
typedef enum LiteRtIntelOpenVinoDeviceType {
43+
kLiteRtIntelOpenVinoDeviceTypeCPU = 0,
44+
kLiteRtIntelOpenVinoDeviceTypeGPU = 1,
45+
kLiteRtIntelOpenVinoDeviceTypeNPU = 2,
46+
kLiteRtIntelOpenVinoDeviceTypeAUTO = 3,
47+
} LiteRtIntelOpenVinoDeviceType;
48+
49+
LiteRtStatus LiteRtIntelOpenVinoOptionsSetDeviceType(
50+
LiteRtIntelOpenVinoOptions options,
51+
enum LiteRtIntelOpenVinoDeviceType device_type);
52+
53+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetDeviceType(
54+
LiteRtIntelOpenVinoOptions options,
55+
enum LiteRtIntelOpenVinoDeviceType* device_type);
56+
57+
// performance_mode -----------------------------------------------------------
58+
59+
// Configures OpenVINO devices to optimize for performance or efficiency.
60+
// See ov::hint::PerformanceMode in OpenVINO. By default, it
61+
// will use LATENCY mode.
62+
63+
typedef enum LiteRtIntelOpenVinoPerformanceMode {
64+
/* Optimize for low latency */
65+
kLiteRtIntelOpenVinoPerformanceModeLatency = 0,
66+
/* Optimize for high throughput */
67+
kLiteRtIntelOpenVinoPerformanceModeThroughput = 1,
68+
/* Optimize for cumulative throughput */
69+
kLiteRtIntelOpenVinoPerformanceModeCumulativeThroughput = 2,
70+
} LiteRtIntelOpenVinoPerformanceMode;
71+
72+
LiteRtStatus LiteRtIntelOpenVinoOptionsSetPerformanceMode(
73+
LiteRtIntelOpenVinoOptions options,
74+
LiteRtIntelOpenVinoPerformanceMode performance_mode);
75+
76+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetPerformanceMode(
77+
LiteRtIntelOpenVinoOptions options,
78+
LiteRtIntelOpenVinoPerformanceMode* performance_mode);
79+
80+
// configs_map ----------------------------------------------------------------
81+
82+
// Set a custom configuration option with a string key-value pair.
83+
// The key and value strings are copied internally, so their lifetime does not
84+
// need to extend beyond this function call.
85+
LiteRtStatus LiteRtIntelOpenVinoOptionsSetConfigsMapOption(
86+
LiteRtIntelOpenVinoOptions options,
87+
const char* key,
88+
const char* value);
89+
90+
// Get the number of custom configuration options
91+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetNumConfigsMapOptions(
92+
LiteRtIntelOpenVinoOptions options,
93+
int* num_options);
94+
95+
// Get a custom configuration option by index.
96+
// The returned key and value pointers point to internal string data
97+
// and are valid for the lifetime of the options object.
98+
// The caller should not free these pointers.
99+
LiteRtStatus LiteRtIntelOpenVinoOptionsGetConfigsMapOption(
100+
LiteRtIntelOpenVinoOptions options,
101+
int index,
102+
const char** key,
103+
const char** value);
104+
105+
#ifdef __cplusplus
106+
107+
} // extern "C"
108+
109+
#endif // __cplusplus
110+
#endif // THIRD_PARTY_ODML_LITERT_LITERT_C_OPTIONS_LITERT_INTEL_OPENVINO_OPTIONS_H_

‎litert/cc/dynamic_runtime/options/BUILD‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ LITERT_C_API_DEPS = [
3535
"//litert/c/options:litert_darwinn_runtime_options",
3636
"//litert/c/options:litert_google_tensor_options",
3737
"//litert/c/options:litert_gpu_options",
38+
"//litert/c/options:litert_intel_openvino_options",
3839
"//litert/c/options:litert_mediatek_options",
3940
"//litert/c/options:litert_qualcomm_options",
4041
"//litert/c/options:litert_runtime_options",
@@ -168,6 +169,27 @@ cc_library(
168169
}),
169170
)
170171

172+
cc_library(
173+
name = "litert_intel_openvino_options",
174+
srcs = ["//litert/cc/options:litert_intel_openvino_options.cc"],
175+
hdrs = ["//litert/cc/options:litert_intel_openvino_options.h"],
176+
deps = [
177+
"//litert/c/options:litert_intel_openvino_options",
178+
"//litert/cc:litert_detail",
179+
"//litert/cc:litert_expected",
180+
"//litert/cc:litert_handle",
181+
"//litert/cc:litert_macros",
182+
"//litert/cc/dynamic_runtime:litert_opaque_options",
183+
"@com_google_absl//absl/strings:string_view",
184+
] + select({
185+
"@org_tensorflow//tensorflow:ios": LITERT_C_API_DEPS,
186+
"@org_tensorflow//tensorflow:emscripten": LITERT_C_API_DEPS,
187+
"//conditions:default": [
188+
"//litert/c:litert_runtime_c_api_shared_lib",
189+
],
190+
}),
191+
)
192+
171193
cc_library(
172194
name = "litert_runtime_options",
173195
srcs = ["//litert/cc/options:litert_runtime_options.cc"],

‎litert/cc/options/BUILD‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,34 @@ cc_library(
166166
alwayslink = True,
167167
)
168168

169+
cc_library(
170+
name = "litert_intel_openvino_options",
171+
srcs = ["litert_intel_openvino_options.cc"],
172+
hdrs = ["litert_intel_openvino_options.h"],
173+
deps = [
174+
"//litert/c:litert_common",
175+
"//litert/c:litert_opaque_options",
176+
"//litert/c/options:litert_intel_openvino_options",
177+
"//litert/cc:litert_detail",
178+
"//litert/cc:litert_expected",
179+
"//litert/cc:litert_handle",
180+
"//litert/cc:litert_macros",
181+
"//litert/cc:litert_opaque_options",
182+
"@com_google_absl//absl/strings:string_view",
183+
],
184+
alwayslink = True,
185+
)
186+
187+
cc_test(
188+
name = "litert_intel_openvino_options_test",
189+
srcs = ["litert_intel_openvino_options_test.cc"],
190+
deps = [
191+
":litert_intel_openvino_options",
192+
"//litert/c/options:litert_intel_openvino_options",
193+
"@com_google_googletest//:gtest_main",
194+
],
195+
)
196+
169197
cc_library(
170198
name = "litert_runtime_options",
171199
srcs = ["litert_runtime_options.cc"],

0 commit comments

Comments
 (0)