Skip to content

[DRAFT] Allow EP to provide additional HW devices #26234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 12 commits into
base: main
Choose a base branch
from
Draft

[DRAFT] Allow EP to provide additional HW devices #26234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
283bd5b
First draft (untested)
adrianlizarraga Sep 16, 2025
bc4df99
Properly discard invalid hw devices provided by the EP; Add hw device…
adrianlizarraga Sep 16, 2025
fb7441c
Add example code to example EP factory
adrianlizarraga Sep 16, 2025
5dafb50
Merge branch 'main' into adrianl/plugin-ep-specify-ort-hw-device
adrianlizarraga Oct 7, 2025
d23bd3a
Stub out default implementation of new factory function for internal eps
adrianlizarraga Oct 8, 2025
f86a4e9
Check for vendor too
adrianlizarraga Oct 9, 2025
f711df7
Formalize hardware metadata keys
adrianlizarraga Oct 9, 2025
02ced24
Remove discrete key
adrianlizarraga Oct 9, 2025
7d30fa7
Make global keys const
adrianlizarraga Oct 9, 2025
f9514a3
Add a new EP for testing
adrianlizarraga Oct 9, 2025
b553858
Rename class
adrianlizarraga Oct 9, 2025
24316d9
Add test for new test EP
adrianlizarraga Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 44 additions & 5 deletions cmake/onnxruntime_unittests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1982,9 +1982,13 @@ endif()
if (WIN32 AND onnxruntime_BUILD_SHARED_LIB AND
NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND
NOT onnxruntime_MINIMAL_BUILD)

#
# example_plugin_ep
file(GLOB onnxruntime_autoep_test_library_src "${TEST_SRC_DIR}/autoep/library/*.h"
"${TEST_SRC_DIR}/autoep/library/*.cc")
#
file(GLOB onnxruntime_autoep_test_library_src "${TEST_SRC_DIR}/autoep/library/example_plugin_ep/*.h"
"${TEST_SRC_DIR}/autoep/library/example_plugin_ep/*.cc"
"${TEST_SRC_DIR}/autoep/library/plugin_ep_utils.h")
onnxruntime_add_shared_library_module(example_plugin_ep ${onnxruntime_autoep_test_library_src})
target_include_directories(example_plugin_ep PRIVATE ${REPO_ROOT}/include/onnxruntime/core/session)
target_link_libraries(example_plugin_ep PRIVATE onnxruntime)
Expand All @@ -1994,12 +1998,12 @@ if (WIN32 AND onnxruntime_BUILD_SHARED_LIB AND
set(ONNXRUNTIME_AUTOEP_LIB_LINK_FLAG "-Xlinker -dead_strip")
elseif (NOT CMAKE_SYSTEM_NAME MATCHES "AIX")
string(CONCAT ONNXRUNTIME_AUTOEP_LIB_LINK_FLAG
"-Xlinker --version-script=${TEST_SRC_DIR}/autoep/library/example_plugin_ep_library.lds "
"-Xlinker --version-script=${TEST_SRC_DIR}/autoep/library/example_plugin_ep/example_plugin_ep_library.lds "
"-Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack")
endif()
else()
set(ONNXRUNTIME_AUTOEP_LIB_LINK_FLAG
"-DEF:${TEST_SRC_DIR}/autoep/library/example_plugin_ep_library.def")
"-DEF:${TEST_SRC_DIR}/autoep/library/example_plugin_ep/example_plugin_ep_library.def")
endif()

set_property(TARGET example_plugin_ep APPEND_STRING PROPERTY LINK_FLAGS
Expand All @@ -2008,7 +2012,42 @@ if (WIN32 AND onnxruntime_BUILD_SHARED_LIB AND
set_target_properties(example_plugin_ep PROPERTIES FOLDER "ONNXRuntimeTest")
source_group(TREE ${TEST_SRC_DIR} FILES ${onnxruntime_autoep_test_library_src})

#
# example_plugin_ep_virt_gpu
#
set(onnxruntime_autoep_test_example_plugin_ep_virt_gpu_src
"${TEST_SRC_DIR}/autoep/library/plugin_ep_utils.h"
"${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep_lib_entry.cc"
"${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep_factory.h"
"${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep_factory.cc"
"${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep.h"
"${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep.cc")
onnxruntime_add_shared_library_module(example_plugin_ep_virt_gpu ${onnxruntime_autoep_test_example_plugin_ep_virt_gpu_src})
target_include_directories(example_plugin_ep_virt_gpu PRIVATE ${REPO_ROOT}/include/onnxruntime/core/session)
target_link_libraries(example_plugin_ep_virt_gpu PRIVATE onnxruntime)

if(UNIX)
if (APPLE)
set(ONNXRUNTIME_AUTOEP_EP_LIB_VIRT_GPU_LINK_FLAG "-Xlinker -dead_strip")
elseif (NOT CMAKE_SYSTEM_NAME MATCHES "AIX")
string(CONCAT ONNXRUNTIME_AUTOEP_EP_LIB_VIRT_GPU_LINK_FLAG
"-Xlinker --version-script=${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep_lib.lds "
"-Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack")
endif()
else()
set(ONNXRUNTIME_AUTOEP_EP_LIB_VIRT_GPU_LINK_FLAG
"-DEF:${TEST_SRC_DIR}/autoep/library/example_plugin_ep_virt_gpu/ep_lib.def")
endif()

set_property(TARGET example_plugin_ep_virt_gpu APPEND_STRING PROPERTY LINK_FLAGS
${ONNXRUNTIME_AUTOEP_EP_LIB_VIRT_GPU_LINK_FLAG})

set_target_properties(example_plugin_ep_virt_gpu PROPERTIES FOLDER "ONNXRuntimeTest")
source_group(TREE ${TEST_SRC_DIR} FILES ${onnxruntime_autoep_test_example_plugin_ep_virt_gpu_src})

#
# test library
#
file(GLOB onnxruntime_autoep_test_SRC "${ONNXRUNTIME_AUTOEP_TEST_SRC_DIR}/*.h"
"${ONNXRUNTIME_AUTOEP_TEST_SRC_DIR}/*.cc")

Expand Down Expand Up @@ -2041,7 +2080,7 @@ if (WIN32 AND onnxruntime_BUILD_SHARED_LIB AND
TARGET onnxruntime_autoep_test
SOURCES ${onnxruntime_autoep_test_SRC} ${onnxruntime_unittest_main_src}
LIBS ${onnxruntime_autoep_test_LIBS}
DEPENDS ${all_dependencies} example_plugin_ep
DEPENDS ${all_dependencies} example_plugin_ep example_plugin_ep_virt_gpu
)
endif()

Expand Down
1 change: 1 addition & 0 deletions include/onnxruntime/core/session/environment.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ class Environment {
~EpInfo();

std::unique_ptr<EpLibrary> library;
std::vector<std::unique_ptr<OrtHardwareDevice>> additional_hw_devices;
std::vector<std::unique_ptr<OrtEpDevice>> execution_devices;
std::vector<OrtEpFactory*> factories;
std::vector<EpFactoryInternal*> internal_factories; // factories that can create IExecutionProvider instances
Expand Down
60 changes: 60 additions & 0 deletions include/onnxruntime/core/session/onnxruntime_ep_c_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,33 @@ struct OrtEpApi {
*/
ORT_API_T(uint64_t, GetSyncIdForLastWaitOnSyncStream,
_In_ const OrtSyncStream* producer_stream, _In_ const OrtSyncStream* consumer_stream);

/** \brief Create an OrtHardwareDevice.
*
* \note Called within OrtEpFactory::GetAdditionalHardwareDevices to augment the list of devices discovered by ORT.
*
* \param[in] type The hardware device type.
* \param[in] vendor_id The hardware device's vendor identifier.
* \param[in] device_id The hardware device's identifier.
* \param[in] vendor_name The hardware device's vendor name as a null-terminated string. Copied by ORT.
* \param[in] metadata Optional OrtKeyValuePairs instance for hardware device metadata that may be queried by
* applications via OrtApi::GetEpDevices() or the EP factory that receives this hardware device
* instance as input to OrtEpFactory::GetSupportedDevices().
* Refer to onnxruntime_ep_device_ep_metadata_keys.h for common OrtHardwareDevice metadata keys.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: does ORT take ownership and EP does not need to release metadata KVP instance if provided?

* \param[out] hardware_device Output parameter set to the new OrtHardwareDevice instance that is created.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.24.
*/
ORT_API2_STATUS(CreateHardwareDevice, _In_ OrtHardwareDeviceType type,
_In_ uint32_t vendor_id,
_In_ uint32_t device_id,
_In_ const char* vendor_name,
_In_opt_ const OrtKeyValuePairs* metadata,
_Out_ OrtHardwareDevice** hardware_device);

ORT_CLASS_RELEASE(HardwareDevice);
};

/**
Expand Down Expand Up @@ -981,6 +1008,39 @@ struct OrtEpFactory {
_In_ const OrtMemoryDevice* memory_device,
_In_opt_ const OrtKeyValuePairs* stream_options,
_Outptr_ OrtSyncStreamImpl** stream);

/** \brief Get additional hardware devices from the execution provider to augment the devices discovered by ORT.
*
* \note Any returned devices that have already been found by ORT are ignored.
*
* \note New additional devices created by this EP factory are not provided to other EP factories. Only this
* EP factory receives the new additional hardware devices via OrtEpFactory::GetSupportedDevices().
* Any OrtEpDevice instances that this EP factory creates with an additional hardware device are visible to
* applications that call OrtApi::GetEpDevices().
Comment on lines +1016 to +1019
Copy link
Contributor

@skottmckay skottmckay Oct 10, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this complexity?

Alternatively, if we're only going to show the OrtEpDevice to the EP that created it, can the EP create it inside of GetSupportedDevices?

e.g. inside GetSupportedDevices

  • EP determines a virtual device is required
  • EP creates OrtHardwareDevice.
  • EP uses that to create OrtEpDevice.

EP could register the OrtHardwareDevice with ORT so we can release it when the EP is unregistered, or EP could own it if we provided a ReleaseHardwareDevice function. Only place the OrtHardwareDevice shows up is indirectly in the OrtEpDevice.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that helps with complexity, but it lacks a signal for the EP to know it should create a virtual device.

GetAdditionalHardwareDevices is better, but that doesn't have a signal for device type (e.g. create a GPU or NPU or both).

Or do we expect it creates all possible virtual devices by default?

*
* \param[in] this_ptr The OrtEpFactory instance.
* \param[in] found_devices Array of hardware devices that have already been found by ORT during device discovery.
* \param[in] num_found_devices Number of hardware devices that have already been found by ORT.
* \param[out] additional_devices Additional OrtHardwareDevice instances that the EP can use.
* The implementation should call OrtEpApi::CreateHardwareDevice to create the devices,
* and then add the new OrtHardwareDevice instances to this pre-allocated array.
* ORT will take ownership of the values returned. i.e. usage is:
* `additional_devices[0] = <OrtHardwareDevice created via CreateHardwareDevice>;`
* \param[in] max_additional_devices The maximum number of OrtHardwareDevice instances that can be added to
* `additional_devices`. Current default is 8. This can be increased if needed.
* \param[out] num_additional_devices The number of additional hardware devices actually added
* to `additional_devices`.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.24.
*/
ORT_API2_STATUS(GetAdditionalHardwareDevices, _In_ OrtEpFactory* this_ptr,
_In_reads_(num_found_devices) const OrtHardwareDevice* const* found_devices,
_In_ size_t num_found_devices,
_Inout_ OrtHardwareDevice** additional_devices,
_In_ size_t max_additional_devices,
_Out_ size_t* num_additional_devices);
};

#ifdef __cplusplus
Expand Down
14 changes: 13 additions & 1 deletion include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

#pragma once

// This file contains well-known keys for OrtEpDevice EP metadata entries.
// This file contains well-known keys for OrtEpDevice and OrtHardwareDevice metadata entries.
// It does NOT specify all available metadata keys.

// Key for the execution provider version string. This should be available for all plugin EPs.
Expand All @@ -16,3 +16,15 @@ static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compa

// Key for the execution provider library path (for dynamically loaded EPs)
static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path";

// Key to retrieve the identity of the entity that discovered and initialized the OrtHardwareDevice.
// Possible values:
// - "ONNX Runtime" (devices discovered by ONNX Runtime).
// - <EP_NAME> (devices discovered by a plugin EP library registered with the OrtEnv).
static const char* const kOrtHardwareDevice_MetadataKey_DiscoveredBy = "DiscoveredBy";

// Key to determine if a OrtHardwareDevice represents a virtual (non-hardware) device.
// Possible values:
// - "0": OrtHardwareDevice is not virtual; represents an actual hardware device.
// - "1": OrtHardwareDevice is virtual.
static const char* const kOrtHardwareDevice_MetadataKey_IsVirtual = "IsVirtual";
5 changes: 5 additions & 0 deletions onnxruntime/core/platform/apple/device_discovery.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <TargetConditionals.h>

#include "core/common/logging/logging.h"
#include "core/session/onnxruntime_ep_device_ep_metadata_keys.h"

namespace onnxruntime {

Expand All @@ -27,6 +28,8 @@ std::vector<OrtHardwareDevice> GetGpuDevices() {
gpu_device.type = OrtHardwareDeviceType_GPU;
gpu_device.vendor_id = kApplePciVendorId;
gpu_device.vendor = kAppleVendorName;
gpu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_DiscoveredBy, "ONNX Runtime");
gpu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_IsVirtual, "0");

result.emplace_back(std::move(gpu_device));
}
Expand Down Expand Up @@ -74,6 +77,8 @@ std::vector<OrtHardwareDevice> GetNpuDevices() {
npu_device.type = OrtHardwareDeviceType_NPU;
npu_device.vendor_id = kApplePciVendorId;
npu_device.vendor = kAppleVendorName;
npu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_DiscoveredBy, "ONNX Runtime");
npu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_IsVirtual, "0");

result.emplace_back(std::move(npu_device));
}
Expand Down
3 changes: 3 additions & 0 deletions onnxruntime/core/platform/device_discovery_common.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include "core/common/cpuid_info.h"
#include "core/common/logging/logging.h"
#include "core/session/onnxruntime_ep_device_ep_metadata_keys.h"

namespace onnxruntime {

Expand Down Expand Up @@ -48,6 +49,8 @@ OrtHardwareDevice DeviceDiscovery::GetCpuDeviceFromCPUIDInfo() {
cpu_device.vendor_id = cpuid_info.GetCPUVendorId();
cpu_device.device_id = 0;
cpu_device.type = OrtHardwareDeviceType_CPU;
cpu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_DiscoveredBy, "ONNX Runtime");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this as well as IsVirtual? Wary of having too much stuff in metadata, and possibly the IsVirtual implies it was not discovered by ORT unless there's some other feature that will rely on an explicit 'discovered by' value.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My thought is that:

  • DiscoveredBy tells the application who discovered the device. May be useful if the app only wants stuff discovered by ORT.
  • IsVirtual is "1" if it does not correspond to actual hardware. It's the negation of IsHardware used by dxcore api. This is a separate concept from DiscoveredBy. This would be useful for off-target compilation, for example.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we only show the virtual device to the EP that created it, is there value in DiscoveredBy?

cpu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_IsVirtual, "0");

return cpu_device;
}
Expand Down
5 changes: 5 additions & 0 deletions onnxruntime/core/platform/linux/device_discovery.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "core/common/logging/logging.h"
#include "core/common/parse_string.h"
#include "core/common/string_utils.h"
#include "core/session/onnxruntime_ep_device_ep_metadata_keys.h"

namespace fs = std::filesystem;

Expand Down Expand Up @@ -138,6 +139,10 @@ Status GetGpuDevices(std::vector<OrtHardwareDevice>& gpu_devices_out) {
for (const auto& gpu_sysfs_path_info : gpu_sysfs_path_infos) {
OrtHardwareDevice gpu_device{};
ORT_RETURN_IF_ERROR(GetGpuDeviceFromSysfs(gpu_sysfs_path_info, gpu_device));

gpu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_DiscoveredBy, "ONNX Runtime");
gpu_device.metadata.Add(kOrtHardwareDevice_MetadataKey_IsVirtual, "0");

gpu_devices.emplace_back(std::move(gpu_device));
}

Expand Down
4 changes: 4 additions & 0 deletions onnxruntime/core/platform/windows/device_discovery.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "core/common/logging/logging.h"
#include "core/platform/env.h"
#include "core/session/abi_devices.h"
#include "core/session/onnxruntime_ep_device_ep_metadata_keys.h"

//// For SetupApi info
#include <Windows.h>
Expand Down Expand Up @@ -618,6 +619,9 @@ std::unordered_set<OrtHardwareDevice> DeviceDiscovery::DiscoverDevicesForPlatfor
std::unordered_map<std::wstring, std::wstring>* extra_metadata = nullptr) {
OrtHardwareDevice ortdevice{device.type, device.vendor_id, device.device_id, to_safe_string(device.vendor)};

ortdevice.metadata.Add(kOrtHardwareDevice_MetadataKey_DiscoveredBy, "ONNX Runtime");
ortdevice.metadata.Add(kOrtHardwareDevice_MetadataKey_IsVirtual, "0");

if (!device.description.empty()) {
ortdevice.metadata.Add("Description", to_safe_string(device.description));
}
Expand Down
Loading
Loading