diff --git a/onnxruntime/test/framework/execution_provider_conformance_test.cc b/onnxruntime/test/framework/execution_provider_conformance_test.cc new file mode 100644 index 0000000000000..96b4bda8635b4 --- /dev/null +++ b/onnxruntime/test/framework/execution_provider_conformance_test.cc @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Execution Provider conformance test suite. +// +// These parameterized tests encode invariants that EVERY IExecutionProvider +// implementation is expected to satisfy, independent of the specific hardware +// backend. They turn the previously-implicit Liskov-substitutability +// assumptions of the IExecutionProvider contract into enforced, executable +// checks, so that a new (or modified) EP cannot silently violate the behavior +// the framework relies on. +// +// Adding an EP to the coverage is a single line: append an entry to +// GetEpConformanceParams() below, guarded by the appropriate USE_* macro. The +// stored value is a *factory*, not a constructed provider, so: +// - No EP is instantiated during static initialization. +// - A factory that returns nullptr (EP compiled but unavailable at runtime, +// e.g. no GPU present) causes the affected test to be skipped, not failed. +// +// Only documented, backend-agnostic contracts are asserted here. Memory that is +// not CPU-accessible is never dereferenced from the test thread; such checks are +// guarded by OrtDevice::UsesCpuMemory(). + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "core/framework/allocator.h" +#include "core/framework/data_transfer.h" +#include "core/framework/data_types.h" +#include "core/framework/execution_provider.h" +#include "core/framework/tensor.h" + +#include "test/util/include/default_providers.h" + +namespace onnxruntime { +namespace test { + +namespace { + +// One EP under test: a human-readable name (also used as the gtest parameter +// suffix, so it must be a valid identifier) plus a factory that constructs a +// fresh provider instance. +struct EpConformanceParam { + std::string name; + std::function()> factory; +}; + +std::vector GetEpConformanceParams() { + std::vector params; + + // CPU is always available. Cover both the arena and non-arena allocator paths + // since they are distinct IAllocator implementations with different Alloc/Free + // behavior. + params.push_back({"Cpu_Arena", [] { return DefaultCpuExecutionProvider(/*enable_arena*/ true); }}); + params.push_back({"Cpu_NoArena", [] { return DefaultCpuExecutionProvider(/*enable_arena*/ false); }}); + +#ifdef USE_CUDA + params.push_back({"Cuda", [] { return DefaultCudaExecutionProvider(); }}); +#endif +#ifdef USE_DML + params.push_back({"Dml", [] { return DefaultDmlExecutionProvider(); }}); +#endif + // Mirror the guard used by base_tester.cc / default_providers.cc: in + // ORT_USE_EP_API_ADAPTERS builds DefaultWebGpuExecutionProvider() ORT_ENFORCEs + // (aborting the whole test run) when the dynamic plugin EP is initialized to a + // different EP, rather than cleanly returning nullptr. Only list the built-in + // WebGPU EP when it is not routed through the EP API adapters. +#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) + params.push_back({"WebGpu", [] { return DefaultWebGpuExecutionProvider(); }}); +#endif +#ifdef USE_XNNPACK + params.push_back({"Xnnpack", [] { return DefaultXnnpackExecutionProvider(); }}); +#endif + + return params; +} + +} // namespace + +class EpConformanceTest : public testing::TestWithParam { + protected: + // Construct the EP under test. Returns nullptr when the EP is compiled but not + // available in the current environment; callers should GTEST_SKIP() in that case. + std::unique_ptr MakeEp() const { return GetParam().factory(); } +}; + +// Invariant: Type() is non-empty and stable -- both across repeated calls on a +// single instance and across independent instances from the same factory. The +// framework keys kernel registries and node assignment on this string. +TEST_P(EpConformanceTest, TypeIsNonEmptyAndStable) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + const std::string type = ep->Type(); + EXPECT_FALSE(type.empty()) << "IExecutionProvider::Type() must not be empty."; + EXPECT_EQ(type, ep->Type()) << "Type() must be stable across calls on the same instance."; + + auto ep2 = MakeEp(); + ASSERT_NE(ep2, nullptr); + EXPECT_EQ(type, ep2->Type()) << "Type() must be identical for instances from the same factory."; +} + +// Invariant: GetPreferredLayout() returns one of the defined DataLayout values. +// Layout transformation dispatches on this, so an out-of-range value is a bug. +TEST_P(EpConformanceTest, PreferredLayoutIsValid) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + const DataLayout layout = ep->GetPreferredLayout(); + EXPECT_TRUE(layout == DataLayout::NCHW || layout == DataLayout::NHWC) + << "GetPreferredLayout() returned an unknown DataLayout value."; +} + +// Invariant: the CPU mem types always map to CPU-accessible memory. The +// framework's input/output staging copies depend on this for every EP. +TEST_P(EpConformanceTest, CpuMemTypesMapToCpuAccessibleDevice) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + EXPECT_TRUE(ep->GetOrtDeviceByMemType(OrtMemTypeCPUInput).UsesCpuMemory()) + << "OrtMemTypeCPUInput must map to CPU-accessible memory."; + EXPECT_TRUE(ep->GetOrtDeviceByMemType(OrtMemTypeCPUOutput).UsesCpuMemory()) + << "OrtMemTypeCPUOutput must map to CPU-accessible memory."; +} + +// Invariant: CreatePreferredAllocators() never yields a null allocator and is +// repeatable. The header documents it as a stateless factory, so a second call +// must produce an equivalently-sized set. +TEST_P(EpConformanceTest, PreferredAllocatorsAreNonNullAndRepeatable) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + auto allocators = ep->CreatePreferredAllocators(); + for (const auto& alloc : allocators) { + EXPECT_NE(alloc, nullptr) << "CreatePreferredAllocators() must not return null entries."; + } + + auto allocators2 = ep->CreatePreferredAllocators(); + EXPECT_EQ(allocators.size(), allocators2.size()) + << "CreatePreferredAllocators() must be repeatable (documented as stateless)."; +} + +// Invariant: each CPU-accessible preferred allocator hands back usable memory: +// a non-zero allocation yields a non-null, host-writable and -readable pointer +// that can be freed. Device allocators are intentionally excluded here -- their +// raw Alloc/Free lifecycle is backend-specific (see body) -- and are covered by +// PreferredAllocatorsAreNonNullAndRepeatable instead. +TEST_P(EpConformanceTest, PreferredAllocatorsAllocateUsableMemory) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + auto allocators = ep->CreatePreferredAllocators(); + if (allocators.empty()) { + GTEST_SKIP() << GetParam().name << " EP exposes no preferred allocators."; + } + + constexpr size_t kBytes = 256; + size_t exercised = 0; + for (const auto& alloc : allocators) { + ASSERT_NE(alloc, nullptr); + + // Standalone Alloc()/Free() is only a backend-agnostic contract for + // CPU-accessible allocators. A device allocator may hand out memory with a + // backend-specific lifecycle that this test cannot drive: e.g. the WebGPU + // GpuBufferAllocator creates buffers mapped at creation that must be + // unmapped through the buffer manager before Free(), so Free()-ing a + // freshly allocated buffer throws. Skip such allocators; they are covered + // by PreferredAllocatorsAreNonNullAndRepeatable. + if (!alloc->Info().device.UsesCpuMemory()) { + continue; + } + + void* p = alloc->Alloc(kBytes); + ASSERT_NE(p, nullptr) << "Alloc(" << kBytes << ") returned null for allocator on " + << alloc->Info().device.ToString(); + + std::memset(p, 0xAB, kBytes); + const auto* bytes = static_cast(p); + EXPECT_EQ(bytes[0], 0xAB); + EXPECT_EQ(bytes[kBytes - 1], 0xAB); + alloc->Free(p); + ++exercised; + } + + if (exercised == 0) { + GTEST_SKIP() << GetParam().name + << " EP exposes no CPU-accessible preferred allocator to exercise."; + } +} + +// Invariant: GetDataTransfer() is optional (may be null). When provided, and it +// advertises the ability to copy within a CPU-accessible device, a CopyTensor +// round-trip must preserve the data exactly. +TEST_P(EpConformanceTest, DataTransferCpuRoundTripPreservesData) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + auto data_transfer = ep->GetDataTransfer(); + if (!data_transfer) { + GTEST_SKIP() << GetParam().name << " EP provides no IDataTransfer (allowed by the contract)."; + } + + // Host the tensors on a CPU-accessible preferred allocator. + AllocatorPtr cpu_alloc; + for (auto& alloc : ep->CreatePreferredAllocators()) { + if (alloc && alloc->Info().device.UsesCpuMemory()) { + cpu_alloc = alloc; + break; + } + } + if (!cpu_alloc) { + GTEST_SKIP() << GetParam().name << " EP has no CPU-accessible allocator to drive the round-trip."; + } + + const OrtDevice& cpu_device = cpu_alloc->Info().device; + if (!data_transfer->CanCopy(cpu_device, cpu_device)) { + GTEST_SKIP() << GetParam().name << " EP data transfer does not advertise CPU<->CPU copy."; + } + + const TensorShape shape({2, 3}); + Tensor src(DataTypeImpl::GetType(), shape, cpu_alloc); + Tensor dst(DataTypeImpl::GetType(), shape, cpu_alloc); + + const float values[] = {1.f, -2.f, 3.5f, 4.f, 5.f, -6.25f}; + constexpr size_t kNumValues = sizeof(values) / sizeof(values[0]); + ASSERT_EQ(static_cast(shape.Size()), kNumValues); + std::memcpy(src.MutableDataRaw(), values, sizeof(values)); + std::memset(dst.MutableDataRaw(), 0, sizeof(values)); + + ASSERT_TRUE(data_transfer->CopyTensor(src, dst).IsOK()) << "CopyTensor failed for a CPU<->CPU copy."; + EXPECT_EQ(std::memcmp(src.DataRaw(), dst.DataRaw(), sizeof(values)), 0) + << "A CPU round-trip CopyTensor must preserve data exactly."; +} + +// Invariant: read-only metadata queries are callable and self-consistent on a +// freshly constructed EP (no session or logger required), and the device-id +// accessor agrees with GetDevice(). +TEST_P(EpConformanceTest, MetadataQueriesAreCallable) { + auto ep = MakeEp(); + if (!ep) GTEST_SKIP() << GetParam().name << " EP not available in this environment."; + + EXPECT_EQ(ep->GetDeviceId(), ep->GetDevice().Id()) + << "GetDeviceId() must agree with GetDevice().Id()."; + + // These must simply be callable without crashing on a bare EP instance. + (void)ep->ConcurrentRunSupported(); + (void)ep->GetTuningContext(); + (void)ep->GetOrtDeviceByMemType(OrtMemTypeDefault); +} + +INSTANTIATE_TEST_SUITE_P( + EpContract, EpConformanceTest, testing::ValuesIn(GetEpConformanceParams()), + [](const testing::TestParamInfo& info) { return info.param.name; }); + +} // namespace test +} // namespace onnxruntime