From 1c81cdda195fd793358bb0c35d479ee9efa35a90 Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Wed, 1 Oct 2025 08:16:43 -0700 Subject: [PATCH 1/6] add tests_to_skip to test dynamic plugin EP configuration --- onnxruntime/test/unittest_main/test_main.cc | 24 ++++++++++++++ .../unittest_util/test_dynamic_plugin_ep.cc | 9 ++++++ .../unittest_util/test_dynamic_plugin_ep.h | 7 +++++ .../util/include/skipping_test_listener.h | 31 +++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 onnxruntime/test/util/include/skipping_test_listener.h diff --git a/onnxruntime/test/unittest_main/test_main.cc b/onnxruntime/test/unittest_main/test_main.cc index 117a26d48efe9..da09bcb8c5e1b 100644 --- a/onnxruntime/test/unittest_main/test_main.cc +++ b/onnxruntime/test/unittest_main/test_main.cc @@ -3,8 +3,10 @@ #include #include +#include #include #include +#include #ifdef _WIN32 #include #include @@ -35,6 +37,7 @@ #if defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) #include "test/unittest_util/test_dynamic_plugin_ep.h" +#include "test/util/include/skipping_test_listener.h" #endif // defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) std::unique_ptr ort_env; @@ -107,6 +110,19 @@ extern "C" void ortenv_teardown() { ort_env.reset(); } +static std::vector> MakeTestEventListeners() { + std::vector> result{}; +#if defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) + { + namespace dynamic_plugin_ep_infra = onnxruntime::test::dynamic_plugin_ep_infra; + const auto tests_to_skip = dynamic_plugin_ep_infra::GetEpTestsToSkip(); + auto skipping_test_listener = std::make_unique(tests_to_skip); + result.emplace_back(std::move(skipping_test_listener)); + } +#endif + return result; +} + #ifdef USE_TENSORRT #if defined(_MSC_VER) @@ -152,6 +168,14 @@ int TEST_MAIN(int argc, char** argv) { ortenv_setup(); ::testing::InitGoogleTest(&argc, argv); + { + auto& test_listeners = ::testing::UnitTest::GetInstance()->listeners(); + auto test_listeners_to_add = MakeTestEventListeners(); + for (auto& test_listener_to_add : test_listeners_to_add) { + test_listeners.Append(test_listener_to_add.release()); + } + } + status = RUN_ALL_TESTS(); } ORT_CATCH(const std::exception& ex) { diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index 6ac741fb616a8..1f79e27bf6ec7 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -90,6 +90,7 @@ Status ParseInitializationConfig(std::string_view json_str, InitializationConfig config.selected_ep_name = parsed_json.value("selected_ep_name", {}); config.selected_ep_device_indices = parsed_json.value("selected_ep_device_indices", {}); + config.tests_to_skip = parsed_json.value("tests_to_skip", {}); config_out = std::move(config); return Status::OK(); @@ -198,4 +199,12 @@ std::optional GetEpName() { return g_plugin_ep_infrastructure_state->ep_name; } +std::vector GetEpTestsToSkip() { + if (!IsInitialized()) { + return {}; + } + + return g_plugin_ep_infrastructure_state->config.tests_to_skip; +} + } // namespace onnxruntime::test::dynamic_plugin_ep_infra diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index 9815eb685f64f..4b49cc8e66974 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -46,6 +46,10 @@ struct InitializationConfig { std::vector selected_ep_device_indices{}; std::map default_ep_options{}; + + // Specifies any tests to skip. + // Tests should be specified by full name, i.e., ".". + std::vector tests_to_skip{}; }; // Parses `InitializationConfig` from JSON. @@ -75,6 +79,9 @@ std::unique_ptr MakeEp(const logging::Logger* logger = nullp // Gets the dynamic plugin EP name, or `std::nullopt` if uninitialized. std::optional GetEpName(); +// Gets the list of tests to skip for the dynamic plugin EP, or an empty list if uninitialized. +std::vector GetEpTestsToSkip(); + } // namespace dynamic_plugin_ep_infra } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/skipping_test_listener.h b/onnxruntime/test/util/include/skipping_test_listener.h new file mode 100644 index 0000000000000..e5405fea620d9 --- /dev/null +++ b/onnxruntime/test/util/include/skipping_test_listener.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include "gsl/gsl" + +#include "gtest/gtest.h" + +namespace onnxruntime::test { + +// A test event listener that skips the specified tests. +class SkippingTestListener : public ::testing::EmptyTestEventListener { + public: + explicit SkippingTestListener(gsl::span tests_to_skip) + : tests_to_skip_(tests_to_skip.begin(), tests_to_skip.end()) { + } + + private: + void OnTestStart(const ::testing::TestInfo& test_info) override { + const auto full_test_name = std::string(test_info.test_suite_name()) + "." + test_info.name(); + if (tests_to_skip_.find(full_test_name) != tests_to_skip_.end()) { + GTEST_SKIP(); + } + } + + std::unordered_set tests_to_skip_; +}; + +} // namespace onnxruntime::test From 2fff53a6f1ba90c390933639324bae35d04a7941 Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Wed, 1 Oct 2025 08:18:11 -0700 Subject: [PATCH 2/6] add check for non-empty provider type in TransformerMemcpyImpl::IsNodeCompatibleWithProvider() --- onnxruntime/core/optimizer/transformer_memcpy.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/optimizer/transformer_memcpy.cc b/onnxruntime/core/optimizer/transformer_memcpy.cc index 46311303639ab..69b024568dfc5 100644 --- a/onnxruntime/core/optimizer/transformer_memcpy.cc +++ b/onnxruntime/core/optimizer/transformer_memcpy.cc @@ -250,8 +250,11 @@ static const IExecutionProvider* FindProviderByType(ProviderTypeToProviderMap pr bool TransformerMemcpyImpl::IsNodeCompatibleWithProvider(const onnxruntime::Node& node) const { const auto& node_provider_type = node.GetExecutionProviderType(); + ORT_ENFORCE(!node_provider_type.empty(), + "Provider type for ", node.OpType(), " node with name '", node.Name(), "' is not set."); const auto* node_provider = FindProviderByType(providers_by_type_, node_provider_type); - ORT_ENFORCE(node_provider != nullptr, "Unable to get provider associated with provider type ", node_provider_type); + ORT_ENFORCE(node_provider != nullptr, + "Unable to get provider associated with provider type '", node_provider_type, "'."); // Same provider? if (node_provider->Type() == provider_.Type()) { From 5cd3f8265cef5797ba34ea7eab9a7dc0784f65bc Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:59:52 -0700 Subject: [PATCH 3/6] add example_plugin_ep test to windows x64 release CI --- .../windows_x64_release_build_x64_release.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/windows_x64_release_build_x64_release.yml b/.github/workflows/windows_x64_release_build_x64_release.yml index f9d7b0d9e9e04..18dd46b4f969e 100644 --- a/.github/workflows/windows_x64_release_build_x64_release.yml +++ b/.github/workflows/windows_x64_release_build_x64_release.yml @@ -94,6 +94,34 @@ jobs: ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' DocUpdateNeeded: 'false' + - name: Run onnxruntime_provider_test with example_plugin_ep + shell: pwsh + run: | + # Note on skipped tests: + # The skipped tests are either: + # - relying on CPU EP fallback for BFloat16 which is not supported + # - testing contrib op LayerNormalization with mixed input/output types (only supported by a few EPs) + # Some other hardcoded EP types are skipped in these tests. For a plugin EP, we skip these tests by + # specifying them in the dynamic plugin EP configuration. + + $env:ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON = @" + { + "ep_library_registration_name": "example_ep", + "ep_library_path": "./example_plugin_ep.dll", + "selected_ep_name": "example_ep", + "tests_to_skip": [ + "LayerNormTest.LayerNorm_BFloat16Input", + "LayerNormTest.LayerNorm_Scale_Float16Input", + "LayerNormTest.LayerNorm_Scale_Float16ScaleOutput", + "LayerNormTest.LayerNorm_Scale_Bias_Float16Input", + "LayerNormTest.LayerNorm_Scale_Bias_Float16ScaleBiasOutput" + ] + } + "@ + + .\onnxruntime_provider_test.exe + working-directory: ${{ github.workspace }}\build\RelWithDebInfo\RelWithDebInfo + - name: Validate C# native delegates shell: cmd run: python tools\ValidateNativeDelegateAttributes.py From 39b29d3271e1a45649c67a3c33ad858e58ebe66b Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Wed, 1 Oct 2025 15:04:10 -0700 Subject: [PATCH 4/6] update comment --- .github/workflows/windows_x64_release_build_x64_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_x64_release_build_x64_release.yml b/.github/workflows/windows_x64_release_build_x64_release.yml index 18dd46b4f969e..013479d86fcce 100644 --- a/.github/workflows/windows_x64_release_build_x64_release.yml +++ b/.github/workflows/windows_x64_release_build_x64_release.yml @@ -100,7 +100,7 @@ jobs: # Note on skipped tests: # The skipped tests are either: # - relying on CPU EP fallback for BFloat16 which is not supported - # - testing contrib op LayerNormalization with mixed input/output types (only supported by a few EPs) + # - testing the LayerNormalization contrib op with mixed input/output types (only supported by a few EPs) # Some other hardcoded EP types are skipped in these tests. For a plugin EP, we skip these tests by # specifying them in the dynamic plugin EP configuration. From 7757f88b729cc9b12c4801e1c5bfe14db59ee607 Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Wed, 1 Oct 2025 15:41:32 -0700 Subject: [PATCH 5/6] add pragma once --- onnxruntime/test/util/include/skipping_test_listener.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onnxruntime/test/util/include/skipping_test_listener.h b/onnxruntime/test/util/include/skipping_test_listener.h index e5405fea620d9..8d631263f6f5f 100644 --- a/onnxruntime/test/util/include/skipping_test_listener.h +++ b/onnxruntime/test/util/include/skipping_test_listener.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#pragma once + #include #include From 3fa25125e6581f89300a3010be75e26ca07927c4 Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Wed, 1 Oct 2025 15:51:06 -0700 Subject: [PATCH 6/6] rename GetEpTestsToSkip to GetTestsToSkip --- onnxruntime/test/unittest_main/test_main.cc | 2 +- onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc | 2 +- onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/unittest_main/test_main.cc b/onnxruntime/test/unittest_main/test_main.cc index da09bcb8c5e1b..a475c0c974f39 100644 --- a/onnxruntime/test/unittest_main/test_main.cc +++ b/onnxruntime/test/unittest_main/test_main.cc @@ -115,7 +115,7 @@ static std::vector> MakeTestEventL #if defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) { namespace dynamic_plugin_ep_infra = onnxruntime::test::dynamic_plugin_ep_infra; - const auto tests_to_skip = dynamic_plugin_ep_infra::GetEpTestsToSkip(); + const auto tests_to_skip = dynamic_plugin_ep_infra::GetTestsToSkip(); auto skipping_test_listener = std::make_unique(tests_to_skip); result.emplace_back(std::move(skipping_test_listener)); } diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index 1f79e27bf6ec7..fd2cf2f712628 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -199,7 +199,7 @@ std::optional GetEpName() { return g_plugin_ep_infrastructure_state->ep_name; } -std::vector GetEpTestsToSkip() { +std::vector GetTestsToSkip() { if (!IsInitialized()) { return {}; } diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index 4b49cc8e66974..680045be9330c 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -79,8 +79,8 @@ std::unique_ptr MakeEp(const logging::Logger* logger = nullp // Gets the dynamic plugin EP name, or `std::nullopt` if uninitialized. std::optional GetEpName(); -// Gets the list of tests to skip for the dynamic plugin EP, or an empty list if uninitialized. -std::vector GetEpTestsToSkip(); +// Gets the list of tests to skip, or an empty list if uninitialized. +std::vector GetTestsToSkip(); } // namespace dynamic_plugin_ep_infra } // namespace test